{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embeddings\n",
    "\n",
    "An embedding is a low-dimensional, vector representation of a (typically) high-dimensional feature which maintains the semantic meaning of the feature in a such a way that similar features are close in the embedding space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import callbacks, layers, models, utils\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow_hub import KerasLayer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding layer for categorical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight_pounds,is_male,mother_age,plurality,gestation_weeks\n",
      "5.2690480617999995,false,15,Single(1),28\n",
      "6.37576861704,Unknown,15,Single(1),30\n",
      "7.7492485093,true,42,Single(1),31\n",
      "1.25002102554,true,14,Twins(2),25\n",
      "8.68841774542,true,15,Single(1),31\n",
      "1.25002102554,Unknown,42,Single(1),23\n",
      "7.50012615324,true,15,Single(1),45\n",
      "6.37576861704,true,15,Single(1),47\n",
      "4.7509617461,true,42,Single(1),25\n"
     ]
    }
   ],
   "source": [
    "!head ./data/babyweight_sample.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Single(1)\n",
       "1    Single(1)\n",
       "2    Single(1)\n",
       "3     Twins(2)\n",
       "4    Single(1)\n",
       "Name: plurality, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./data/babyweight_sample.csv\") \n",
    "df.plurality.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Single(1)', 'Twins(2)', 'Triplets(3)', 'Multiple(2+)',\n",
       "       'Quadruplets(4)'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.plurality.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLASSES = {\n",
    "    'Single(1)': 0,\n",
    "    'Multiple(2+)': 1,\n",
    "    'Twins(2)': 2,\n",
    "    'Triplets(3)': 3,\n",
    "    'Quadruplets(4)': 4,\n",
    "    'Quintuplets(5)': 5\n",
    "}\n",
    "N_CLASSES = len(CLASSES)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the plurality to a numeric index."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "plurality_class = [CLASSES[plurality] for plurality in df.plurality]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    Single(1)\n",
      "1    Single(1)\n",
      "2    Single(1)\n",
      "3     Twins(2)\n",
      "4    Single(1)\n",
      "Name: plurality, dtype: object\n",
      "[0, 0, 0, 2, 0]\n"
     ]
    }
   ],
   "source": [
    "print(df.plurality[:5])\n",
    "print(plurality_class[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create an embedding layer. Supply arguments `input_dim` and `output_dim`\n",
    " - `input_dim` indicates the size of the vocabulary. For `plurality` this is 6.\n",
    " - `ouptut_dim` indicates the dimension of the dense embedding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBED_DIM = 2\n",
    "\n",
    "embedding_layer = layers.Embedding(input_dim=N_CLASSES,\n",
    "                                   output_dim=EMBED_DIM)\n",
    "embeds = embedding_layer(tf.constant(plurality_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The variable `embeds` contains the two-dimensional for each plurality class. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([999, 2])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 2), dtype=float32, numpy=\n",
       "array([[ 0.00296154,  0.04679434],\n",
       "       [ 0.00296154,  0.04679434],\n",
       "       [ 0.00296154,  0.04679434],\n",
       "       [ 0.03186324, -0.00946026],\n",
       "       [ 0.00296154,  0.04679434]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeds[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding Layers in a Keras model\n",
    "\n",
    "In this section, we will implement text models to recognize the probable source (Github, Tech-Crunch, or The New-York Times) of the titles we have in the title dataset we constructed in the previous lab.\n",
    "\n",
    "In a first step, we will load and pre-process the texts and labels so that they are suitable to be fed to a Keras model. For the texts of the titles we will learn how to split them into a list of tokens, and then how to map each token to an integer using the Keras Tokenizer class. What will be fed to our Keras models will be batches of padded list of integers representing the text. For the labels, we will learn how to one-hot-encode each of the 3 classes into a 3 dimensional basis vector.\n",
    "\n",
    "Then we will explore a few possible models to do the title classification. All models will be fed padded list of integers, and all models will start with a Keras Embedding layer that transforms the integer representing the words into dense vectors.\n",
    "\n",
    "Our model will be a simple bag-of-words DNN model that averages up the word vectors and feeds the tensor that results to further dense layers. Doing so means that we forget the word order (and hence that we consider sentences as a “bag-of-words”). Using an RNN or a 1-dimensional CNN would allow us to maintain the order of word embeddings in our model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by specifying where the information about the trained models will be saved as well as where our dataset is located:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOGDIR = \"./text_models\"\n",
    "DATA_DIR = \"./data\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our dataset consists of titles of articles along with the label indicating from which source these articles have been taken from (GitHub, Tech-Crunch, or the New-York Times)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>holy cash cow  batman - content is back</td>\n",
       "      <td>nytimes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>show hn  a simple and configurable deployment ...</td>\n",
       "      <td>github</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>show hn  neural turing machine in pure numpy. ...</td>\n",
       "      <td>github</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>close look at a flu outbreak upends some commo...</td>\n",
       "      <td>nytimes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>lambdalite  a functional  relational lisp data...</td>\n",
       "      <td>github</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title   source\n",
       "0            holy cash cow  batman - content is back  nytimes\n",
       "1  show hn  a simple and configurable deployment ...   github\n",
       "2  show hn  neural turing machine in pure numpy. ...   github\n",
       "3  close look at a flu outbreak upends some commo...  nytimes\n",
       "4  lambdalite  a functional  relational lisp data...   github"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATASET_NAME = \"titles_full.csv\"\n",
    "TITLE_SAMPLE_PATH = os.path.join(DATA_DIR, DATASET_NAME)\n",
    "COLUMNS = ['title', 'source']\n",
    "\n",
    "titles_df = pd.read_csv(TITLE_SAMPLE_PATH, header=None, names=COLUMNS)\n",
    "titles_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we'll find how many words we have in our dataset (`VOCAB_SIZE`), how many titles we have (`DATASET_SIZE`), and what the maximum length of the titles we have (`MAX_LEN`) is. Keras offers the `Tokenizer` class in its `keras.preprocessing.text` module to help us with this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(titles_df.title)\n",
    "integerized_titles = tokenizer.texts_to_sequences(titles_df.title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The variable 'integerized_titles' contains the integer representation of each article title in out dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[6117, 560, 8577, 13948, 302, 13, 172],\n",
       " [11, 12, 2, 49, 7, 3838, 1322, 91, 4, 28, 482],\n",
       " [11, 12, 1501, 2812, 322, 5, 589, 7337, 5458, 78, 108, 1989, 17, 1139]]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "integerized_titles[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From this and the `tokenizer` we can extract the `VOCAB_SIZE`, `DATASET_SIZE` and `MAX_LEN`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47271"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VOCAB_SIZE = len(tokenizer.index_word)\n",
    "VOCAB_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96203"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATASET_SIZE = tokenizer.document_count\n",
    "DATASET_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_LEN = max(len(sequence) for sequence in integerized_titles)\n",
    "MAX_LEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data\n",
    "We'll need to pad the elements of our title to feed into the model. Keras has the helper functions `pad_sequence` for that on the top of the tokenizer methods.\n",
    "\n",
    "The function `create_sequences` will \n",
    "* take as input our titles as well as the maximum sentence length and \n",
    "* returns a list of the integers corresponding to our tokens padded to the sentence maximum length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_sequences(texts, max_len=MAX_LEN):\n",
    "    sequences = tokenizer.texts_to_sequences(texts)\n",
    "    padded_sequences = pad_sequences(sequences,\n",
    "                                     max_len,\n",
    "                                     padding='post')\n",
    "    return padded_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6117,   560,  8577, 13948,   302,    13,   172,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0],\n",
       "       [ 1030,   316,    23,     2,  3718,  7338, 13949,   214,   715,\n",
       "         4581,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "            0,     0,     0,     0,     0,     0,     0,     0]],\n",
       "      dtype=int32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_titles = create_sequences([\"holy cash cow  batman - content is back\",\n",
    "                                 \"close look at a flu outbreak upends some common wisdom\"])\n",
    "sample_titles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll convert our label to numeric, categorical variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLASSES = {\n",
    "    'github': 0,\n",
    "    'nytimes': 1,\n",
    "    'techcrunch': 2\n",
    "}\n",
    "N_CLASSES = len(CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_labels(sources):\n",
    "    classes = [CLASSES[source] for source in sources]\n",
    "    one_hots = utils.to_categorical(classes)\n",
    "    return one_hots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create train/validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_TRAIN = int(DATASET_SIZE * 0.80)\n",
    "\n",
    "titles_df = pd.read_csv(TITLE_SAMPLE_PATH, header=None, names=COLUMNS)\n",
    "titles_train, sources_train = (\n",
    "    titles_df.title[:N_TRAIN], titles_df.source[:N_TRAIN])\n",
    "\n",
    "titles_valid, sources_valid = (\n",
    "    titles_df.title[N_TRAIN:], titles_df.source[N_TRAIN:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "github        29175\n",
       "techcrunch    24784\n",
       "nytimes       23003\n",
       "Name: source, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources_train.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, prepare the data for the model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, Y_train = create_sequences(titles_train), encode_labels(sources_train)\n",
    "X_valid, Y_valid = create_sequences(titles_valid), encode_labels(sources_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 6117,   560,  8577, 13948,   302,    13,   172,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0],\n",
       "        [   11,    12,     2,    49,     7,  3838,  1322,    91,     4,\n",
       "            28,   482,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0],\n",
       "        [   11,    12,  1501,  2812,   322,     5,   589,  7337,  5458,\n",
       "            78,   108,  1989,    17,  1139,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0]],\n",
       "       dtype=int32),\n",
       " array([[0., 1., 0.],\n",
       "        [1., 0., 0.],\n",
       "        [1., 0., 0.]], dtype=float32))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[:3], Y_train[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build a DNN model\n",
    "\n",
    "The `build_dnn_model` function below returns a compiled Keras model that implements a simple embedding layer transforming the word integers into dense vectors, followed by a Dense softmax layer that returns the probabilities for each class.\n",
    "\n",
    "\n",
    "Note that we need to put a custom Keras Lambda layer in between the Embedding layer and the Dense softmax layer to do an average of the word vectors returned by the embedding layer. This is the average that's fed to the dense softmax layer. By doing so, we create a model that is simple but that loses information about the word order, creating a model that sees sentences as \"bag-of-words\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dnn_model(embed_dim):\n",
    "\n",
    "    model = models.Sequential([\n",
    "        layers.Embedding(VOCAB_SIZE + 1,\n",
    "                         embed_dim,\n",
    "                         input_shape=[MAX_LEN]),\n",
    "        layers.Lambda(lambda x: tf.reduce_mean(x, axis=1)),\n",
    "        layers.Dense(N_CLASSES, activation='softmax')\n",
    "    ])\n",
    "\n",
    "    model.compile(\n",
    "        optimizer='adam',\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(76962, 3)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 76962 samples, validate on 19241 samples\n",
      "Epoch 1/100\n",
      "76962/76962 [==============================] - 3s 43us/sample - loss: 1.0480 - accuracy: 0.4290 - val_loss: 0.9778 - val_accuracy: 0.5822\n",
      "Epoch 2/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.8877 - accuracy: 0.6848 - val_loss: 0.8051 - val_accuracy: 0.7366\n",
      "Epoch 3/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.7342 - accuracy: 0.7832 - val_loss: 0.6821 - val_accuracy: 0.7891\n",
      "Epoch 4/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.6254 - accuracy: 0.8138 - val_loss: 0.5948 - val_accuracy: 0.8104\n",
      "Epoch 5/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.5453 - accuracy: 0.8310 - val_loss: 0.5322 - val_accuracy: 0.8197\n",
      "Epoch 6/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.4865 - accuracy: 0.8433 - val_loss: 0.4882 - val_accuracy: 0.8241\n",
      "Epoch 7/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.4429 - accuracy: 0.8520 - val_loss: 0.4559 - val_accuracy: 0.8355\n",
      "Epoch 8/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.4092 - accuracy: 0.8603 - val_loss: 0.4322 - val_accuracy: 0.8391\n",
      "Epoch 9/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.3825 - accuracy: 0.8676 - val_loss: 0.4138 - val_accuracy: 0.8420\n",
      "Epoch 10/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.3604 - accuracy: 0.8743 - val_loss: 0.3996 - val_accuracy: 0.8466\n",
      "Epoch 11/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.3419 - accuracy: 0.8791 - val_loss: 0.3886 - val_accuracy: 0.8486\n",
      "Epoch 12/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.3258 - accuracy: 0.8842 - val_loss: 0.3800 - val_accuracy: 0.8508\n",
      "Epoch 13/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.3117 - accuracy: 0.8891 - val_loss: 0.3726 - val_accuracy: 0.8516\n",
      "Epoch 14/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2990 - accuracy: 0.8931 - val_loss: 0.3672 - val_accuracy: 0.8543\n",
      "Epoch 15/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2877 - accuracy: 0.8971 - val_loss: 0.3624 - val_accuracy: 0.8546\n",
      "Epoch 16/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2771 - accuracy: 0.9005 - val_loss: 0.3587 - val_accuracy: 0.8562\n",
      "Epoch 17/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2675 - accuracy: 0.9038 - val_loss: 0.3560 - val_accuracy: 0.8577\n",
      "Epoch 18/100\n",
      "76962/76962 [==============================] - 2s 26us/sample - loss: 0.2585 - accuracy: 0.9072 - val_loss: 0.3541 - val_accuracy: 0.8583\n",
      "Epoch 19/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2502 - accuracy: 0.9103 - val_loss: 0.3521 - val_accuracy: 0.8597\n",
      "Epoch 20/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2422 - accuracy: 0.9131 - val_loss: 0.3513 - val_accuracy: 0.8599\n",
      "Epoch 21/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2347 - accuracy: 0.9157 - val_loss: 0.3508 - val_accuracy: 0.8607\n",
      "Epoch 22/100\n",
      "76962/76962 [==============================] - 2s 27us/sample - loss: 0.2277 - accuracy: 0.9179 - val_loss: 0.3512 - val_accuracy: 0.8605\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 26, 10)            472720    \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 3)                 33        \n",
      "=================================================================\n",
      "Total params: 472,753\n",
      "Trainable params: 472,753\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "CPU times: user 1min 35s, sys: 11.4 s, total: 1min 47s\n",
      "Wall time: 46.8 s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xUVf7/8deZTHrvIYUk9JJIC6gIwQpYUUEBu1+/utZ1/dp197fuum5xV9d1V9d1XduuBRTsiGVBAUUlgWBCCYSSThpJSK/n98cdQiaNQAamfZ6Pxzxm5p47M5/MI7xzOPfcc5XWGiGEEM7PZO8ChBBC2IYEuhBCuAgJdCGEcBES6EII4SIk0IUQwkWY7fXBEREROikpyV4fL4QQTikzM7NSax3ZV5vdAj0pKYmMjAx7fbwQQjglpVR+f20y5CKEEC5CAl0IIVyEBLoQQrgIu42hCyHcU1tbG0VFRTQ3N9u7FIfm4+NDfHw8np6eg36NBLoQ4qQqKioiMDCQpKQklFL2Lschaa2pqqqiqKiI5OTkQb9OhlyEECdVc3Mz4eHhEuYDUEoRHh5+zP+LkUAXQpx0EuZHdzzfkdMFel55Hb/+aDut7Z32LkUIIRyK0wV64cEmXv5mH2tzy+1dihDCSQUEBNi7hBPC6QJ99ugIIgO9eTezyN6lCCGEQ3G6QDd7mLh8Shxrd5ZTWd9i73KEEE5Ma839999PSkoKqampLFu2DIDS0lLS09OZPHkyKSkprF+/no6ODm644Yauff/85z/bufrenHLa4qJp8fxj3V4+yCrhplmDn9IjhHAsv/poG9tLDtn0PSfEBvHLiycOat+VK1eSlZXF1q1bqaysZPr06aSnp/Pmm28yb948Hn30UTo6OmhsbCQrK4vi4mJycnIAqKmpsWndtuB0PXSA0dGBTEoIkWEXIcSQbNiwgaVLl+Lh4UF0dDRz5sxh06ZNTJ8+nVdeeYXHHnuM7OxsAgMDGTFiBHv37uWuu+5i9erVBAUF2bv8Xpyyhw5GL/0X7+ewraSWibHB9i5HCHEcBtuTPlG01n1uT09PZ926dXzyySdce+213H///Vx33XVs3bqVzz77jOeee47ly5fz8ssvn+SKB+aUPXSAS06JxcvDxDsZ0ksXQhyf9PR0li1bRkdHBxUVFaxbt44ZM2aQn59PVFQUN998MzfddBObN2+msrKSzs5OFi5cyOOPP87mzZvtXX4vTttDD/bz5LyJ0XyQVcwjF4zHy+y0f5uEEHZy2WWXsXHjRiZNmoRSiieffJKYmBhee+01/vjHP+Lp6UlAQACvv/46xcXF3HjjjXR2GufA/O53v7Nz9b2p/v7LcaKlpaXpoV7gYm1uOTe+sokXrpnG/JQYG1UmhDiRduzYwfjx4+1dhlPo67tSSmVqrdP62v+o3Vql1MtKqXKlVE4/7Uop9axSKk8p9aNSaupxVX4cZo+KIErmpAshBDC4MfRXgfkDtJ8PjLbcbgH+PvSyBsfsYeKyqXGszS2nok7mpAsh3NtRA11rvQ44OMAuC4DXteE7IEQpNcxWBR7NoqnxdHRqPsgqPlkfKYQQDskWRxLjgMJuz4ss23pRSt2ilMpQSmVUVFTY4KOt56Tb63iAEEI4AlsEel9rPPaZrFrrF7XWaVrrtMjISBt8tGHRtHh2Hqhjm43POBNCCGdii0AvAhK6PY8HSmzwvoN2eE66HBwVQrgzWwT6h8B1ltkupwG1WutSG7zvoHWfky7rpAsh3NVgpi2+BWwExiqlipRSNymlblVK3WrZZRWwF8gD/gncfsKqBSjZAiv+F9qarDYvmhZPdWMba3aWndCPF0K4l4HWTt+/fz8pKSknsZqBHfVMUa310qO0a+AOm1V0NM2HIPsdGHchTLysa3P66Eiig4w56fNTTtokGyGEcBjOd+p/0iwIiIHsd60C3cOkuGxKPP9cv5eKuhYiA73tWKQQYlA+fQgOZNv2PWNS4fzf99v84IMPkpiYyO23G4MJjz32GEop1q1bR3V1NW1tbfzmN79hwYIFx/Sxzc3N3HbbbWRkZGA2m3n66ac566yz2LZtGzfeeCOtra10dnayYsUKYmNjufLKKykqKqKjo4Nf/OIXLF68eEg/Njjj4lwmDyPId38BzbVWTYumxcmcdCHEgJYsWdJ1IQuA5cuXc+ONN/Lee++xefNm1q5dy7333nvM06Cfe+45ALKzs3nrrbe4/vrraW5u5oUXXuDuu+8mKyuLjIwM4uPjWb16NbGxsWzdupWcnBzmzx/o3M3Bc74eOkDqIvj+77DjY5hyddfmUVGBTE4I4Z2MIm6alSxXFhfC0Q3Qkz5RpkyZQnl5OSUlJVRUVBAaGsqwYcO45557WLduHSaTieLiYsrKyoiJGfwaURs2bOCuu+4CYNy4cSQmJrJr1y5OP/10nnjiCYqKirj88ssZPXo0qamp3HfffTz44INcdNFFzJ492yY/m/P10AHipkFoEuS826tp0bR4cstkTroQon+LFi3i3XffZdmyZSxZsoQ33niDiooKMjMzycrKIjo6mubm5mN6z/569FdddRUffvghvr6+zJs3jzVr1jBmzBgyMzNJTU3l4Ycf5te//rUtfiwnDXSlIGUh7P0a6q3POL34lFi8zDInXQjRvyVLlvD222/z7rvvsmjRImpra4mKisLT05O1a9eSn59/zO+Znp7OG2+8AcCuXbsoKChg7Nix7N27lxEjRvDTn/6USy65hB9//JGSkhL8/Py45ppruO+++2y2trpzBjpA6hWgO2D7+1abg/08mTshmvezimlp77BTcUIIRzZx4kTq6uqIi4tj2LBhXH311WRkZJCWlsYbb7zBuHHjjvk9b7/9djo6OkhNTWXx4sW8+uqreHt7s2zZMlJSUpg8eTI7d+7kuuuuIzs7mxkzZjB58mSeeOIJfv7zn9vk53Lq9dB5fiZ4B8BNn1tt/iq3nBte2cQL10yVKYxCOBhZD33wbL4eukNLXQiF30NNgdXm2d3mpAshhLtw7kBPWWjc56yw2nx4Tvra3ApZJ10IMWTZ2dlMnjzZ6nbqqafau6xenHPa4mGhSRA/HbJXwKx7rJoWTYvjha/38P6WYm5OH2Gf+oQQfdJaO9W04tTUVLKysk7qZx7PcLhz99ABUhZBWTZU5FptPjwnXdZJF8Kx+Pj4UFVVJf8uB6C1pqqqCh8fn2N6nXP30ME4a/Szh42lAM5+1Kpp0bR4fv5+DjnFh0iND7ZTgUKI7uLj4ykqKsJWF7lxVT4+PsTHxx/Ta5w/0AOjITndWLDrrEeMOeoWF58Sy68/3s67mYUS6EI4CE9PT5KTk+1dhkty/iEXMIZdqvdBifXk/GA/T+ZNjOGDrSUyJ10I4fJcI9DHXwweXsbB0R4WTYunprGNNTvK7VCYEEKcPK4R6L4hMOo82LYSOq174rNGRcicdCGEW3CNQAfjJKO6Usj/1mqzh0lx+dR4vtpVQXndsS22I4QQzsR1An3M+eDp3+cKjAunxhvrpG85qdeuFkKIk8p1At3LD8ZdANveh/ZWq6ZRUQFMGS5z0oUQrs11Ah2MFRiba2DPml5Nh9dJzymWddKFEK7JtQJ9xFngG9rnsMtFlnXS38kstENhQghx4rlWoJu9YMIC2LkKWhutmoJ9PTk/JYb3txTT3CZz0oUQrse1Ah2Mk4zaGmDXp72aFk9P4FBzO5/mlNqhMCGEOLFcL9ATZ0LgsD5PMjotOZzEcD/e/kGGXYQQrsf1At3kARMvh7wvoKnausmkuDItge/3HWRvRb2dChRCiBPD9QIdjJOMOlphx0e9mq6YFo+HSbEsQ3rpQgjX4pqBHjsVQpONJXV7iAry4exxUazILKKto9MOxQkhxInhmoGulDEnff96qCvr1bxkegKV9a38VxbsEkK4ENcMdIDURaA7Ydt7vZrmjIkkJsiHtzcV9PFCIYRwTq4b6JFjITq1z5OMzB4mrkiL5+tdFZTUNNmhOCGEsD3XDXQwDo4WbYKD+3o1XZmWgNbwToYsqyuEcA2uHegpC437nN5z0hPC/Jg9OoLlGYV0dMqCXUII5+fagR4yHBJO7TPQwThztLimiQ15lSe5MCGEsD3XDnQwlgIo3w5l23s1nTchmlA/T5bJwVEhhAtw/UCfeCkojz4PjnqbPbh8ajxfbC+jsr7FDsUJIYTtuH6gB0TBiDnGsEsfF7dYMj2Btg7Ne5uL7VCcEELYjusHOhjDLtX7oTizV9Po6ECmJYby1qYCuZqREMKpuUegj78IPLz7XAoAjIOjeysayMiv7rNdCCGcgXsEuk8wjD4Ptq2Ezt4Xt7jolGEEeJtlWV0hhFMbVKArpeYrpXKVUnlKqYf6aA9WSn2klNqqlNqmlLrR9qUOUeoiqC8z1nfpwc/LzCWTY/kku4TapjY7FCeEEEN31EBXSnkAzwHnAxOApUqpCT12uwPYrrWeBJwJPKWU8rJxrUMzZj54BcDWt/tsXjI9gea2Tj7cWnKSCxNCCNsYTA99BpCntd6rtW4F3gYW9NhHA4FKKQUEAAeBdptWOlSevjD5Ksh+B6rzezWnxgUzYViQzEkXQjitwQR6HNB9cLnIsq27vwHjgRIgG7hba91rsXGl1C1KqQylVEZFRcVxljwEZ9wNKPjmmV5NSimWzEggp/gQOcW1J782IYQYosEEuupjW8/5ffOALCAWmAz8TSkV1OtFWr+otU7TWqdFRkYec7FDFhwPU66GLf+BQ72HVhZMisPbbJJldYUQTmkwgV4EJHR7Ho/RE+/uRmClNuQB+4BxtinRxmbdY8x0+eYvvZqC/Ty5IHUYH2wpoam192wYIYRwZIMJ9E3AaKVUsuVA5xLgwx77FADnACilooGxwF5bFmozoUkwaSlkvtrv1YzqWtpZlV160ksTQoihOGqga63bgTuBz4AdwHKt9Tal1K1KqVstuz0OzFRKZQP/BR7UWjvuEoaz/8+4iPTGv/ZqmpEcRnKEvwy7CCGcjnkwO2mtVwGremx7odvjEmCubUs7gcJHGssBbPoXnPEz8I/oalJKsXh6Ar//dCd55fWMigqwY6FCCDF47nGmaF/S74O2Jtj4XK+mhVPjMZsUyzPkzFEhhPNw30CPHAsTFsAP/4TGg9ZNgd6cOz6aFZlFtLb3mn0phBAOyX0DHSD9fmitg+//0atp8YwEqhpa+XJH7wOnQgjhiNw70GNSYNxF8P3fofmQVVP66Ehig314e5MMuwghnIN7BzoYY+nNtfDDi1abPUyKK9ISWL+7gqLqRjsVJ4QQgyeBHjsFRs81Do621Fs1XZEWD8DyjCJ7VCaEEMdEAh0g/QFoOggZ/7LaHB/qx+zRkbyTUUhHp1zNSAjh2CTQARKmw4gz4du/Qqv18MrS6QmU1jazbrcdFhMTQohjIIF+WPoD0FABm1+z2nzO+GjC/b1YJlczEkI4OAn0w5LOgMRZxqJdbc1dm73MJhZOi+fLHWWU1jbZsUAhhBiYBHp3c+6HulLY8m+rzdeelohJKZ7+fJedChNCiKOTQO8ueQ7Ez4ANz0B7a9fmhDA/rp+ZyLubi9hRemiANxBCCPuRQO9OKZjzABwqgq1vWTXdedZognw8+e2qHXYqTgghBiaB3tOoc4256Rueho4jl0UN9vPkrrNHsX53JV/vkhkvQgjHI4Hek1LGGi/V+40LSndz7emJDA/z43erdsi8dCGEw5FA78vYCyA6Fdb/ybhcnYW32YMH5o9l54E6VmyWs0eFEI5FAr0vShlrvFTlwbb3rJouTB3G5IQQnvo8l8bW9n7eQAghTj4J9P6MvwQix8G6P0HnkTXRlVL8/MLxlB1q4V/r99mxQCGEsCaB3h+TCWbfBxU7YOdHVk1pSWHMnxjDC1/voaKuxU4FCiGENQn0gaRcDmEjYd0fQVsfBH3w/HG0tHfyzJdyspEQwjFIoA/E5AGz74UD2bBrtVVTcoQ/15yWyNubCskrr7NTgUIIcYQE+tGcciWEJMLXT/bqpf/0nNH4eXrw+0932qk4IYQ4QgL9aDw8jV56yWbIetOqKczfi9vPGsWXO8rZuKfKTgUKIYRBAn0wplwDiWfA6oegpsCq6cYzkogN9uG3q3bQKScbCSHsSAJ9MEwecOnzoDvh/dutpjH6eHpw37yxZBfX8uHWEjsWKYRwdxLogxWaBPN/D/vXw/cvWDVdOjmOibFB/PGzXJrbOvp+vRBCnGAS6MdiyjUw5nz48jEoP3Ig1GRSPHrBeIprmnj12/12K08I4d4k0I+FUnDJs+AdAO/9BDrauppmjorg7HFRPLc2j+qG1gHeRAghTgwJ9GMVEAUX/wVKs4wTjrp5+PxxNLS08+ya3XYqTgjhziTQj8f4i2HSUmOdl6LMrs2jowNZPH04/96Yz/7KBjsWKIRwRxLox+v8P0DgMHjvFmht7Np8z3mj8TKbePIzOdlICHFySaAfL59gYypjVZ5xkNQiKtCHn6SPZFX2ATLzD9qvPiGE25FAH4oRc+DU2+CHf8CetV2bb05PJirQmyc+2YHWcrKREOLkkEAfqnN/CRFj4IM7oKkGAD8vM/fOHcPmgho+zTlg5wKFEO5CAn2oPH3hsn9A3QH49IGuzYumJTA2OpA/rN5Ja3vnAG8ghBC2IYFuC3FTYc4D8OMy2PY+AB4mxcMXjCO/qpHXN+63a3lCCPcggW4rs++F2Cnw8T1Gbx2YMyaSs8dF8eRnuWwvOWTnAoUQrk4C3VY8POGyF6GtET78KWiNUoonF51CqJ8nd7y5mfoWuai0EOLEkUC3pcgxcO6vYPdnsPk1ACICvHl2yRTyqxp4ZGW2zHoRQpwwgwp0pdR8pVSuUipPKfVQP/ucqZTKUkptU0p9bdsynciMWyB5Dqx+BA7uA+DUEeHcO3csH24t4c0fCo7yBkIIcXyOGuhKKQ/gOeB8YAKwVCk1occ+IcDzwCVa64nAFSegVudgMhknHJnM8P5t0Gksp3vbnJGkj4nkVx9tZ1tJrZ2LFEK4osH00GcAeVrrvVrrVuBtYEGPfa4CVmqtCwC01uW2LdPJBMfDBU9CwUb49q+AscTun6+cRKifJ3e+uYW65rajvIkQQhybwQR6HFDY7XmRZVt3Y4BQpdRXSqlMpdR1fb2RUuoWpVSGUiqjoqLi+Cp2FqcsNhbxWvsEHMgBIDzAm78unUp+VQMPy3i6EMLGBhPoqo9tPZPIDEwDLgTmAb9QSo3p9SKtX9Rap2mt0yIjI4+5WKeiFFz0F/AJgbeXQm0xADOSw7h37lg+/rFUxtOFEDY1mEAvAhK6PY8Hel48swhYrbVu0FpXAuuASbYp0Yn5h8NVy4wlAV6/BOrKAOvx9JxiGU8XQtjGYAJ9EzBaKZWslPIClgAf9tjnA2C2UsqslPIDTgV22LZUJxU3Fa5+Fw6VwusLoKGqx3j6ZhlPF0LYxFEDXWvdDtwJfIYR0su11tuUUrcqpW617LMDWA38CPwAvKS1zjlxZTuZ4afCVW9D9T7496XQVNM1nl5Y3STj6UIIm1D2CpK0tDSdkZFhl8+2m91fwltLIHYyXPseeAfy/Fd5PLk6l99cmsI1pyXau0IhhINTSmVqrdP6apMzRU+m0efCFa9C8WZ4cwm0NnJr+kjmjInk1x/LeLoQYmgk0E+28RfB5S9C/jew7GpMna08feUkwvy8ZDxdCDEkEuj2kLoIFvwN9qyBd24g3NfEX6+aQmF1Ew/JeLoQ4jhJoNvLlGvggj9B7ipYeTPThwdz79wxfPJjKf/5XuanCyGOndneBbi1GTdDezN8/nMw+3DrJc/xw76DPP7RdqYkhJASF2zvCoUQTkR66PY28y4461HY+hamVffy9BWTCPP34g4ZTxdCHCMJdEeQfj/MugcyXyFsw2P8delkiqqbeGiFjKcLIQZPhlwcgVJwzi+hrRm+e57pnn7cN/cq/rB6J+EfevHYxRMxmfpaUkcIIY6QQHcUSsH830F7E6z/E7ee5cPB2Rfyz/X7aOvo5IlLUyXUhRADkkB3JErBhX+GtmbU2t/wyFxfPM88l+e/2kNbh+YPC0/BQ0JdCNEPCXRHYzLBguegvQn1+aPcn7YX37Nu4Km1hbR1dPLUFZMwe8ihDyFEbxLojsjDDAv/BSGJqG+f5a6o7wif9RiPbCihvVPzzOLJeEqoCyF6kFRwVB6eMPdxuHoF1JdxVdZ1vDZ5B5/8WMKdb26mtb3T3hUKIRyMBLqjG30u3PYNJMxgzs7HWZP4Ot9u28dt/8mkpb3D3tUJIRyIBLozCIyBa9+Hc/4fI8q/5NvQX1KV+y03v55Jc5uEuhDCIIHuLEwmmH0v3PgpgV4erPT5NRP2vsxNr3xPY2u7vasTQjgACXRnM/xUuHU9pnEX8JD5LX5S9AB3v/Q59S0S6kK4Owl0Z+QbAle+Dhf9mTPMu/ht2a089fcXZO0XIdycBLqzUgrS/gePn3yFT1Akv6x5lM//chu19Y32rkwIYScS6M4uegKBd62ncMRiFja9Q+kzZ1JbvNveVQkh7EAC3RV4+ZFw3YvkzHyWuLYCzP9Mp37jK9ApM2CEcCcS6C4kZe717Lh0Fbk6noDPfkbTs6fBzlUgS/AK4RYk0F3MjClT8bjpc37h9QClB+vg7aXol+dDwXf2Lk0IcYJJoLugScNDeeD/HuBv4//NI203UV28C16eB28ugbLt9i5PCHGCSKC7qEAfT55eOp20hf/Hee3P8Beuom3fBvj7THjvNqgptHeJQggbk0B3cZdPjefdn57Ll+FXM73uT6yPXIzOWQF/nQafPQqNB+1dohDCRiTQ3UByhD8rbpvJ4vRJXFt4CVf7Pk/NqAXw3fPwl0mw7o/Q2mDvMoUQQySB7ia8zCYevmA8r//PDHY1hzBj2+V8MPMddNIZsOY38OwU2PQSdMjZpkI4Kwl0N5M+JpLVP5vNzJHh3P3fFm5uvY9DSz+GsBHwyb3wtzT45lloqLR3qUKIYySB7oYiArx5+frp/OKiCXy9q5y5K1rZmP4fWLoMAqLhi1/AU+PgnRtgz1rolItpCOEMlLbTSSdpaWk6IyPDLp8tjsgpruWnb21hX1UDd5w5irvPHY1nVS5sfh22vgVN1RCSCFOvg8lXQ9Awe5cshFtTSmVqrdP6bJNAFw0t7fzqo20szyhiyvAQHl+QQkpcMLQ1w86PIfNV2L8elAeMmQdTr4fR54HJw96lC+F2JNDFoHy0tYT/90EONU1tLJwaz31zxxIT7GM0Vu0xeu1Zb0JDOQTGwpRrYOq1EDLcvoUL4UYk0MWg1Ta18fzaPF75Zj8eJsXN6SP4SfoI/L3Nxg4dbZD7KWx+DfL+a2wbeTZMux7GnA9mL/sVL4QbkEAXx6zwYCN/WL2Tj38sJSrQm/vmjWXh1Hg8TOrITjUFsOU/xu1QMXgHGxe1HnsBjDrXuBCHEMKmJNDFccvMr+Y3n2xnS0EN44cF8fMLx3PGqAjrnTo7YM8a2PEh5K42hmRMZkg8wwj3sedDaKJ9fgAhXIwEuhgSrTUf/1jK7z/dSXFNE+eMi+LhC8YzKiqg986dnVCcAbmrjKV7K3ON7dEpR8I9dopxxSUhxDGTQBc20dzWwavf7ue5NXk0tnVw9anDufuc0YQHePf/oqo9RrjnfgoFG0F3GgdUx86HsRdC8mwwD/B6IYQVCXRhU1X1LTzz5W7e/KEAP08P7jx7FDeckYS3+SjTGBuqYPfnkPsJ5K2BtgbwCoCRZ0HiLBh+GsSkynRIIQYw5EBXSs0H/gJ4AC9prX/fz37Tge+AxVrrdwd6Twl055dXXsdvV+1kzc5y4kN9+dm5Y7hkUixe5kGcgNzWDPvWGb33vC+h1rKcr1cgJEyH4acbAR+XBl5+J/YHEcKJDCnQlVIewC7gPKAI2AQs1Vpv72O/L4Bm4GUJdPexYXclT6zawY7SQ0QGenP96YlcfWoiof7HMIWxphAKvzeGZfI3Qvl2QBsHV4dNNsJ9+OnGzT/8hP0sQji6oQb66cBjWut5lucPA2itf9djv58BbcB04GMJdPeitWb97kpe2rCPdbsq8PE0sXBqPP8zK5mRkX0cPD2apmoo3GQEfMFGKM6EjlajLWKMJeBnQtxUCB8lwzTCbQwU6OZBvD4O6H55myLg1B4fEAdcBpyNEej9FXILcAvA8OFydqErUUqRPiaS9DGR5B6o4+UN+3gns4g3vi/gnHFR3DQ7mdNHhKMGO7vFNxTGzDVuYAzRlGZB/rfG9VG3f2CcuQpg9oGo8cZMmphU4z56osyDF25nMD30K4B5Wuv/tTy/Fpihtb6r2z7vAE9prb9TSr2K9NAFUFnfwn++y+ffG/Opamhl/LAg/ndWMhcPdpx9IJ2dULETSrdCWQ4cyDbuG6uO7BM8HGJSLEFvuQ9NBpMsMiqc1wkfclFK7QMOd70igEbgFq31+/29rwS6+2hu6+CDrGJeWr+P3eX1RAV6c/3MJK6aMfzYxtmPRmuoO2Ad8AdyoGq3MV0SjFk1UROMHnz4KGMd+PCREJok0yeFUxhqoJsxDoqeAxRjHBS9Smu9rZ/9X0V66KIPWmvW7a7kpfV7Wb+7Eh9PE4umxXPjGcc5zj5YbU3GQdaybUbAl+UYj5truu2kIDgBwkcYIR82UsJeOKQhjaFrrduVUncCn2FMW3xZa71NKXWrpf0Fm1YrXJZSijljIpnTbZx9+aYi/vNdAdOTQrlsSjwXnjKMYF9P236wpy/ETTNu3TUehIN7jVvVHsvjPbDtPeOg7JHKjbAPSz4S8MEJxjrxIQngHylnvgqHICcWCbuqqGvhncxCVm4uJq+8Hi+zifPGR3P51DjSx0Ti6WGn8e7Gg3BwnxHwPQPfKuwxDsoGJxjhHjLcOuxDhkNAjIzbC5uRM0WFw9Nak1N8iBWbi/hwawkHG1oJ9/fiksmxLJwaz8TYoMHPkDnRmg8ZJ0LVFBjz52vyLc8t2xp7XI/V5AnBcUbQB0SBfxQERBo9+67HUUabDO2Io5BAF06lraOTr3MrWLmliC+3l9Pa0cnoqAAunxrPpVNiGRbsa+8SB9baALVFPcK+AGqLjZUo6yugta7v13oHd9zoeL8AAAx3SURBVAv7yCN/APwjwC/ccm957Bcm8+/dkAS6cFq1jW18kl3Kys1FZORXoxScMTKCy6fGMW9izJELbzibtiaoL4eGCsu9JegbKro9LjfarA7edqeM+fpdQR/e47Hl3ifYcgsy7s0+MubvxCTQhUvYX9nAe1uKWbmliMKDTfh5eXD2uCjOmxDNmWOjbH8w1VG0txrz6xurjOGchkpjjL/rcdWR2+HnuqP/9/PwAu+g3kHvE2zZHmI89vI/cvP06/uxh4t+5w5MAl24FK01GfnVrNxczBfbD1BZ34rZpDhtRDjnTYjm3AnRxIU4+LDMidTZCS21xuqWjVXQcgiaa3vfrLZ3e9zeNPjPMnn2DnpPP/D0Mf4ncPjW87nZ25h9ZPYG8+H77vv1s93D2+0PMEugC5fV2anZUljDF9vL+GL7AfZUNAAwYVgQ502I5rwJ0Y51QNUZtLcawd5aD22NxjGB1oY+Hjd226fH4/ZmaG8x/ji0txhDTO0txvbOtqHV5+HVO+wPB73JDMrDuDd5WG7dt/WxjzL1vlltVz3aPY5s15393PTAbaPOgQkLjuvHl0AXbmNvRb0l3MvILKhGa4gL8eXc8VGcNyGGGclhQ192QAxNRzt0tBjr87R3u1k9b7FuO/y8rbn3H4vDzzs7oLPdGG7q7OjxvN34n4vVc8s+VmHbMXAoH96fbrnZ1x+E/v4YYHk+438h/f7j+vok0IVbqqxvYc2Ocj7fXsaGvAqa2zoJ9DFz5tgozh0fxezRkYTZcukB4T60Nm52GP6RQBdur6m1gw15lXyx/QD/3VFOVYOxFO/E2CBmjY5g1qgIpieF4eMp0wCFY5NAF6Kbjk7Nj0U1bNhdyYa8SjYXVNPWofEym5ieFMqsUZHMGhXBxNggTCYZexeORQJdiAE0tLTzw/6DRsDvriS3zDjpJ9TPk5kjI7p68Alhcik8YX9DvcCFEC7N39vMWWOjOGtsFADldc18m1fF+t2VbMir4JPsUgASw/04Y1QEp48IZ3pSGDHBPvYsW4hepIcuxAC01uypqGf97kq+yatk454qGlqNk3biQ32ZnhRGWlIo05PCGBUZIEM04oSTIRchbKSto5MdpYfYtL+ajP0H2bS/msr6FgCCfT1JSwxlmiXgU+OC5SCrsDkJdCFOEK01+VWNbNp/kIz91WTkH+w6ucnLw8Qp8cGkJYUxPSmUaYmhhPjJNEkxNBLoQpxEVfUtZOZXk5Ffzab9B8kuqqW90/h3lhzhz6T4YCYlhDApIYQJw4KkFy+OiQS6EHbU1NrB1qIaMvOr2VpYQ1ZhDeV1xjCN2aQYPyyISQnBTIoPYXJCCCMiA/CQsXjRDwl0IRzMgdpmsgpr2FpUw9bCGn4sqqW+pR2AAG8zqXFGL35ygnEfE+Qj69EIQKYtCuFwYoJ9mB8cw/yUGMBYZGxvZT1ZhbVstQT9vzbspa3D6HBFBHgxflgQE2KDmBgbzIRhQSRH+EtPXliRHroQDqq5rYMdpYfYWljDtpJDbC89xK6yuq6Q9/X0YNywQCZ0C/qx0YH4esmYvCuTIRchXERreyd55fVsLz3EtpJatluCvq7ZGK4xKRgRGcDE2KCuoB8bE0hkgLcM2bgICXQhXJjWmqLqpq5e/HZL0JfUNnftE+bvxdjoQMbGBDIuxrgfEx3ovJfwc2Myhi6EC1NKkRDmR0KYX9eYPEB1Qys7Sg+x80AduQfq2FlWx/KMQhpbj1yeLiHMl7HRQV0hPy4mkOQIf8wesma8M5JAF8JFhfp7MXNUBDNHRXRt6+w0evM7DxzqCvncA3WszS2nwzJX3svDxMioAMZEBzAqMoCRUQGMigogKdxfLg7i4CTQhXAjJpNieLgfw8P9mDvxSG++ua2DPRX17Cqr6+rRZ+yv5oOskq59PEyKxDA/RkYFMDLSCPlRUQGMjPQn0EcuFu0IJNCFEPh4ejAxNpiJscFW2xtb29lb0cCeinryyo/cvsot75ptAxAT5MPIKH9GWYI+OSKAEZH+xAT5yIJlJ5EEuhCiX35eZlLigkmJsw769o5OCg42GgFvCfs95fWs2FzcdYIUGFMrkyL8GRHhz4hIf5Ij/BkRGUByhD/BvtKrtzUJdCHEMTN7mBgRGcCIyADmdtuutabsUAt7K+vZV9nA3ooG9lU2sK2kltXbDnSN04NxslRyhHXIJ0f4MzzMT9a3OU4S6EIIm1FKERPsQ0ywDzNHRli1tbYbvXoj6I8E/pqdFSzPKLLaNybIh+HhfiSG+ZFkCfnEcD8Sw6VnPxAJdCHESeFlNnUdSIVoq7bapjb2VTaQX9VAflUj+VWNFBxs4KtdFVRkWod9iJ8niWFGuCeG+1nC3ngcGeDt1mP2EuhCCLsL9vVkcoKx2mRPDS3tFBw8EvL7qxopqGpkS2E1H/9YQrdRHLzNJhLCjJC3uoX7kRDq5/LLIkigCyEcmr+3mfHDghg/LKhXW2t7J8U1TeyvaqDoYCMFXbcmvt975HKBh0UGepNoCfmEbvdxob7EBPk4/WJnEuhCCKflZTZ1HUztSWvNwYbWrpAv7Bb43+2t4r2sYrqvfGI2GeP/8aG+xIf6ERfiS3yoL3GhviSE+hET7IOng59BK4EuhHBJSinCA7wJD/BmyvDQXu0t7R0UVzdRZLkV1zR2Pd6wu5KyumarwDcp42BtfKhfV9DHhhi3uBAfhgX72n1tHAl0IYRb8jZ7dE297EtLewelNc0U1zRRVG2E/eE/AN/treLAoWar8XswDtjGBh8J+dgQX4Z1exwVeGKHdSTQhRCiD95m46SopD6GcwDaOjopO9RMaW0zJTVNFNc0UVLTRElNM0XVjXy/r6prWePDzCZFdJAPN8xM4ub0ETavWQJdCCGOg6eHyTL84tfvPoea2yitORL4pbVG4EcFeZ+QmiTQhRDiBAny8SQoxpOxMYEn5fMc+5CtEEKIQRtUoCul5iulcpVSeUqph/pov1op9aPl9q1SapLtSxVCCDGQowa6UsoDeA44H5gALFVKTeix2z5gjtb6FOBx4EVbFyqEEGJgg+mhzwDytNZ7tdatwNvAgu47aK2/1VpXW55+B8TbtkwhhBBHM5hAjwMKuz0vsmzrz03Ap0MpSgghxLEbzCyXvmbB6z62oZQ6CyPQZ/XTfgtwC8Dw4cMHWaIQQojBGEwPvQhI6PY8HijpuZNS6hTgJWCB1rqqrzfSWr+otU7TWqdFRkYeT71CCCH6MZhA3wSMVkolK6W8gCXAh913UEoNB1YC12qtd9m+TCGEEEejtO5z9MR6J6UuAJ4BPICXtdZPKKVuBdBav6CUeglYCORbXtKutU47yntWdNv/WEUAlcf5Wnch39HA5Ps5OvmOBmav7ydRa93nEMegAt3RKKUyjvYHw93JdzQw+X6OTr6jgTni9yNnigohhIuQQBdCCBfhrIEuZ6IenXxHA5Pv5+jkOxqYw30/TjmGLoQQojdn7aELIYToQQJdCCFchNMF+tGW8hWglNqvlMpWSmUppTLsXY+9KaVeVkqVK6Vyum0LU0p9oZTabbnvfRVhN9LPd/SYUqrY8nuUZTkfxS0ppRKUUmuVUjuUUtuUUndbtjvU75FTBfogl/IVhrO01pMdbZ6snbwKzO+x7SHgv1rr0cB/Lc/d2av0/o4A/mz5PZqstV51kmtyJO3AvVrr8cBpwB2W7HGo3yOnCnQGsZSvED1prdcBB3tsXgC8Znn8GnDpSS3KwfTzHQkLrXWp1nqz5XEdsANj1VmH+j1ytkA/1qV83ZUGPldKZVpWuBS9RWutS8H4xwpE2bkeR3Wn5UpkL9t7OMFRKKWSgCnA9zjY75GzBfqgl/J1c2doradiDE3doZRKt3dBwin9HRgJTAZKgafsW479KaUCgBXAz7TWh+xdT0/OFuiDWsrX3WmtSyz35cB7GENVwlqZUmoYgOW+3M71OBytdZnWukNr3Qn8Ezf/PVJKeWKE+Rta65WWzQ71e+RsgX7UpXzdnVLKXykVePgxMBfIGfhVbulD4HrL4+uBD+xYi0M6HFQWl+HGv0dKKQX8C9ihtX66W5ND/R453ZmifS3la+eSHIpSagRGrxyMK1K96e7fkVLqLeBMjOVOy4BfAu8Dy4HhQAFwhdbabQ8K9vMdnYkx3KKB/cBPDo8Xuxul1CxgPZANdFo2P4Ixju4wv0dOF+hCCCH65mxDLkIIIfohgS6EEC5CAl0IIVyEBLoQQrgICXQhhHAREuhCCOEiJNCFEMJF/H+UrxOM6lS6NgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZicVaHn8e/pvau39JKll3R3gED2JgsJgkAkiKhAAEFALyO5ChcHGJZ7BzSK4OhlmIuOgxeBiQrII8h4gQhBFGUNXiHSSUgCCYGY9JbO0qne1+qqOvPHW11dvSWd0J3qeuv3eZ563rXeOl3p/HJy3nPOa6y1iIhI7EuIdgFERGRsKNBFRFxCgS4i4hIKdBERl1Cgi4i4RFK0PrigoMCWl5dH6+NFRGLSxo0bD1lrJw93LGqBXl5eTmVlZbQ+XkQkJhljqkc6piYXERGXUKCLiLiEAl1ExCUU6CIiLqFAFxFxCQW6iIhLKNBFRFwiav3QRUTcyB8I0t7jp7XLT2t3L63dvbR1+2ntCi27e1lclstZM4cdG/SJKNBFRAax1tLa7ae500dTZy/NnT6aO3tpGrTd1t1La7ffWXY5yw5f4IjX/+byExXoIiJHKxi0tHT10tjpo6nDR2OHj6ZOH40doYDu6A/pplBQN3f1EgiO/PCf7LQkJnlSyE5PIjstmYKCDLLTkslOTyYrzdmXlZY0YNs5nkRmahJJiePT2q1AF5GY0uMP0Njhw9vu41B7D952H96OHiegO3w0dvpo7uwLbieoR8rm1KQE8jJSmORJIdeTzKxp2eR4ksn1JJPrcfZPSk8mNyM5dE4KOenJJCaY4/tDj5ICXUSiqq95o6Gtm0PtvnBAH2rr4VCHD284tJ0Ab+v2D3ud5ERDrieFvAwneGdNyyY3I5k8Twq5Gf378zJC254U0lMSj/NPO75GFejGmAuAB4BE4BfW2vsGHc8FHgVOBLqBf7TWvj/GZRWRGNIbCHKovYeGth4OtvbQ0Lfe1h1aOtsNbT30+IND3m8M5HlSyM9MIT8jlblF2RRkppKfkUJ+Zir5mSkUhI7lZaaQlZqEMROz5ny8HDHQjTGJwM+AzwJ1wLvGmBestdsjTlsNvGetvdQYMyt0/orxKLCIRE8gaGnq9NHQ1sOh9p5wYB9q79/XF9aNHb5hr5HrSWZyVipTstIoL89gSlYqk/temanhsM71pEzYpo2JajQ19KXALmvtbgBjzNPASiAy0OcA/xPAWvuhMabcGDPVWntgrAssImPLWktbj5+Drf2154a2/hp1ZFh723uGbY9OTUqgINMJ5ZJcD4vKcpmcmcqU7NTQMo0pWU5Qpya5q5ljIhlNoBcDtRHbdcCyQedsAS4D/mKMWQqUASXAgEA3xlwPXA9QWlp6jEUWkdEIBG1EjbnbafaIaOo42NZNQ7vTHDJck0dKYgIFmSkUZKVSlJPGguIcJmelUpCZwuSstNAylYKsVDV3TBCjCfTh/pQG/xt9H/CAMeY9YBuwGRhy58JauwZYA7BkyZKR+wSJyGH1hXV9cxf7W7qpb+lmf0tXaNnNvuYuDrT1DNv1Lic9OdzMsbg0N9z84Sz7mz9y0pMV0jFmNIFeB0yP2C4B6iNPsNa2AqsAjPMbsCf0EpGjFAhavO09/SHd3M3+1u5weO9r6eZAazf+QWGdmpRA0aR0pmWncfqJ+RTmpDEtJ50pg4JaTR7uNZpAfxeYaYyZAewFrgK+EnmCMWYS0Gmt9QHfANaHQl5EIvTVrPeFatH7WkYX1ilJCRTmpFGYk8ayGXlMy0mjcFI6hdlpFE5KozAnnVyPatTx7oiBbq31G2NuAl7G6bb4qLX2A2PMDaHjjwCzgSeMMQGcm6VfH8cyi0xoLZ29VDd2UO3tpKaxk2qvs17X1DVizdqpTUeEdY4T0n3reRkpCms5olH1Q7fWvgS8NGjfIxHrbwMzx7ZoIhNTMGjZ39odCmwnrKsbO6nxOuHdOmjgy+SsVMryPCydkUfRJKcZRDVrGQ8aKSoyDH8gyN7mLqpCIV11KLT0dlDb1IUvoldIUoKhJDed0vwMKqbnUJaXQWm+h7J8D6V5Hjwp+msmx4d+0yRu+fxBapsGB7azrGvqGtA0kp6cSFm+h5OmZHLe7KlOYOdlUJbvoTAnbdwmWxI5Ggp0cbXW7l5qwm3ZzrKvmaS+uWvAIJms1CTKCjzMLc7hiwsKKcvPoDw/g/J8D5OzUtUsIhOeAl1iWjBoOdDW7bRfh9qxaxr71jto6uwdcH5eRgqleR4Wl+Vy2aISyvM9oeD26MajxDwFusQMb3sP2/e1smNfK9vrW9mxr40qb8eAUY6JCYaiSWmU5WVwwbxCyvI9lOV5KA21Z2elJUfxJxAZXwp0mXACQcueQx1OcEcE+MG2nvA507LTmF2YxdknF1Can+GEdp6H4tx0ktWeLXFKgS5R1enzh4K7LVTrbuXD/a109zq17qQEw0lTMvn0SQXMKcpmdqHzystIiXLJRSYeBbocN929AXbsa2Xb3ha21rWwra6Fjw+2hW9M5qQnM7swi68sLWN2YRZzirI5aUqmhqqLjJICXcaFzx/kowNtTnDvbWZrXQs797eFuwLmZ6SwoCSHz82bxvziHOYUZVOUk6abkiKfgAJdPrFg0PLxwXa21DazdW8z2+pa2LGvDV/AaTaZ5ElmfnEO/3TOCcwvnsSCkhwKFd4y3qwFGxzmNWh/0A+BXgj2QsAfWvZG7Is4FvD1r9sAYJxHK5nQfRuTELFt+rcj100C5JZDwdgPrlegy1Hr7g2wpbaZyuomKqsa2VjdFB7unpWaxLziHFadWc78khwWFE9iel66wtvNggHwdUBvZ+jV7YRe0B8KSP8w2xGvQG//ur/beX9vJ/R2gb/LWUa+wvtCn9Xb5bwvMqSHzPA9wZx5K3z2+2N+WQW6HJG3vScc3pXVTby/t4XegPMXZuaUTL64oJAlZXmcWjqJGfkZJOixYROP3we+dicEfZ3Q2+EsfR39672d/cHs6witdw08Hn5/Z//xQM+RP/9oJSRBsgeS0iA53VlPTnOW6bmQVRjal+68klLBJA6sBYdfEdsMcywhCRKTISE5tEyCxJSI9Yhjket9tXJrATuw9j/stu0/N3Pq2H9nKNBlEGstuw91sLGqiXdDte/dhzoAZwrXipIcvv7pEzitPJdFpbnkqrfJsQv4obsZOr3Q2Qhdjc6y0xuxHtrfVwMN9tVCA6HtQP/2cMeCASeQg0OeN3N4yR7nleKB5AwnNFMyIHNaaN+g4yl94ZrhBG84HBOd9XAwJjrr4X1Joe3Qqy/AEzVe4Fgo0ONcIGjZsa+VDXsa+dseL5VVTXhDD/fN9SSzuCyPK0+bzpLyXOYV58RPjxNrnRptdwt0tzrLntCyt2tgO2vAF2o68IVekW2toeOBXieUw8Htda41ksQUSM8DTz548sBT4NQIExL7a5fh9cSIbeNs9x3rq+lGBm9K5jBhHNqf4oGkdEhQX/5YpECPMz5/kG17m0MB3sjGqibaepzaW2meh+WnTGHpjFwWl+Vx4uSM2Gz77gvjnjYnjHvaoKdl0Hbr0KDubh64zw59zuZhmcT+/6onJjvrkf9VT0p1Qjq3zAnq9DwnrPuW4fV8pzYci9+9RJUC3eW6fAE21zSFA3xTTVN4qPzJUzO5+NQils7IY+mMPApz0qNc2sMI+KF1LzTXQHM1NFU7653e/oDuC2xf2+jCODUb0nKcV2o2ZJfAlOz+7bQcSIvcnuRsJ6dHBHVKf7uqarUSZQp0l/EHgry928t/7vLytz1etta14A9aEgzMLcrhH04vY+mMPE4rz5tYoy2DQWg/4IR1c00osKv6g7t176B2YAPZxZA5GVKzIOOEUOhmO9upoWVadmg9cjsLUrIUwOI6CnQXCAYt71Y1sm5rPS9t209jh4/kRENFySSuP/sEls7IY3FZ7vGfmMrf49SgOw5B5yHo8IZuAB7q39fZGAry2qG9JTKnwqQyKDkNci931ieVOk0W2SWQNIH+QRKZABToMcpay5a6FtZtqef3W/exv7WbtOQEzps9lYsqijh75mTSU8bhBmag1wngtv2h177Q9j4npCPD29c2/DVMgtNWnFHg3OybOg9O+UIorMtDwT3dadoQkVFToMcQay07D7Sxbks967bso6axk5TEBM45ZTKrK2azYtYUMlI/wR+p3wcHtkHrvoFB3bYf2kLrnYeGvs8kQMYUyJzi3NDLm+EEdUZ+qJdGQX94ZxQ4bdIJcdJbRuQ4UqDHgN0N7by4dR/rttTz8cF2EhMMZ55UwM3nnsT5c6eRk/4JmlJ8HbDrVdixDj562ekN0sckOM0eWdMgpwRKljjrWdOcgR2ZU51lRoECWmQCUKBPUI0dPv6jspZ1W+t5f28rxsBp5Xn88JJ5fH7eNPIzU4/94p2N8NEfYceL8PdXnf7R6bkw+0KYeb7T7KGgFok5CvQJ5kBrN2vW7+apDTV09QaomD6J735xNl9cUPjJuhW21sOHv3dq4lV/cUYTZhfDoq85QV56hjNqT0Rilv4GTxC1jZ08/ObfeaayjoC1rKwo4oblJ3Ly1Kxjv+ihXfDhOqcmvrfS2Zc/E868xQnxokUavCLiIgr0KNt1sI2HXv87z2+pJ9EYLl9Swg1nn0hpvmf0F+luDQ24qYGWWmiqgr+/Dg07nONFC+Hcu2D2RTD5lHH5OUQk+hToUfL+3hZ+9vou/vjBftKSErn2jHKuO+sEpuWkDT05MrDDr+r+9e7mgecnpTs3MBf/L5j1RacLoIi4ngL9OKusauTB13fxxs4GslKTuHH5Saw6s7z/JmdnI1S9BXvWQ927zkjJwYGd7HH6bE8qhelLnWXO9P6BNxkFakoRiUMK9OPAWstfdh3iwdd2sWFPI3kZKfz3z53CNZ8qI9v0QM162PMm7H4T9m8DrDML3vSlzijJvvCeVOqEtidfgS0iQyjQx9lbHzfwo5d3sqWuhanZqdz9hZO4unA/abX/D55c79ysDPqdSZ5KlsJnVsOMs6F4seaEFpGjokAfJ9Za/u/63fzbH7azIqee3y3Yy4LeLSSs3+A8QsskODcrz7jZCfDppztzUYuIHCMF+jjoDQS5a+02mjY9x7uZvyG/5wB8BEyZA4u/BjPOgbIzIH1StIsqIi6iQB9jLV29/PDxtVxc/wBnpbyPzZsDn/5XOOEcZ64TEZFxMqpAN8ZcADwAJAK/sNbeN+h4DvBroDR0zR9Zax8b47JOeLX7DvD2o3dwr28dNi0Dzvs3zJKvawSmiBwXR0waY0wi8DPgs0Ad8K4x5gVr7faI024EtltrLzLGTAZ2GmOetNb6xqXUE4217HntUbLe+h9cTgsNM69g6iX3Og9fEBE5TkZTdVwK7LLW7gYwxjwNrAQiA90CWcZ5AGUm0Agc5WPGY9S+LTT+x63MaNzEjoSZ9Fz2a4rnnRXtUolIHBpNoBcDtRHbdcCyQec8CLwA1ANZwJXWDn2oozHmeuB6gNLS0mMp78TR2Yh97YfYyscI2kweyrmVq67/NnmZw4z0FBE5DkYT6MONYLGDtj8HvAecC5wI/NkY85a1tnXAm6xdA6wBWLJkyeBrxIZgADY+jn3tBwS7WnnC/1k+mnMz93z5DFKTNNWsiETPaAK9DoicDKQEpyYeaRVwn7XWAruMMXuAWcDfxqSUE0XNBnjpX2D/VranzOf2nq/yhRXnce+KkzAauSkiUTaaQH8XmGmMmQHsBa4CvjLonBpgBfCWMWYqcAqweywLGlXdrfCHO2DLb/BnFHJv6r/w6/bF3H9lBStPLY526UREgFEEurXWb4y5CXgZp9vio9baD4wxN4SOPwL8AHjcGLMNp4nmTmvtMA+fjFG//2d4/1nq5/9XvvT+6XSbdJ68bgmnledFu2QiImGj6iBtrX0JeGnQvkci1uuB88e2aBPE9udh22/ZccqNrNx0FiV56Tx97WmU5WdEu2QiIgNoxMvhtB+EdbfSljePi7Ys47QTcnnkHxaT49GkWSIy8SjQR2ItrLsFfB38KP82JmVm8Kt/XEpKUkK0SyYiMiyl00i2/AZ2vkTnWat5ao+HlacWKcxFZEJTQg2npQ7+cCeUnsFzKRfRG7Bctki9WURkYlOgDxYMwvM3OgOILnmIZ9/bzylTs5hTmB3tkomIHJYCfbDKX8LuN+BzP2RPcAqba5q5dFGxBg6JyISnQI/k/Tv8+Xtw0nmweBVrN+/FGFh5alG0SyYickQK9D7BAPzum85zPC/+dyywdnMdZ55YQGFOerRLJyJyRAr0Pn/9d6jdAF/4EWQXUVndRG1jF5cu1M1QEYkNCnSAAx/A6/8Ksy+G+VcA8NymvaQnJ3LBvGlRLpyIyOgo0P0+WPtPkJYDF/4EjKG7N8Dvt9bzublTyUjV2CsRiQ1Kq/X3w/5tcNVTkFEAwOsfHqS1289li0qiXDgRkdGL7xr63o3w1o+h4isw64vh3c9u2suUrFTOPKkgioUTETk68RvovV2w9gbIKoTP3xfe3djh442dB1l5ahGJCep7LiKxI36bXF79ARz6CK75ndN+HvLi1nr8QculC9XcIiKxJT5r6FV/gXcegtOugxM/M+DQc5v2MmtaFnOKNNRfRGJL/AV6T5szgChvBnz2+wMO/b2hnfdqmzURl4jEpPhrcnn5O85siqv+CCkDnzr0u817STDoOaEiEpPiq4b+0Z9g06/gjP8GpcsGHAoGLWs37+XMkwqYmp0WpQKKiBy7+Ar0P30XJs+Gz6wecqiyuom6pi41t4hIzIqfQG8/CId2wqlXQ1LqkMPPbarDk5LI5+ZqqL+IxKb4CfTaDc5y+ulDDnX3Bvj9tn1cMHcanpT4u60gIu4QP4Fe8w4kpkLRqUMOvbrjIG3dfi5Vc4uIxLD4CvTiRcM2t6zdXMfU7FTOOFFD/UUkdsVHoPs6Yd8WKB3a3OJt7+GNnQ1ccmqxhvqLSEyLj0Cv3wTB3mHbz9dtCQ31V3OLiMS4+Aj0mnec5fSlQw6t3byX2YXZzJqmof4iEtviJ9AnzwJP3oDdf29oZ0tdC5fpMXMi4gLuD/RgEGr/BtOXDTm0dlPfUP+iKBRMRGRsuT/QG3ZATwuUfmrA7r6h/p+eOZkpGuovIi7g/kDvaz8fNHfL36oa2dvcpeYWEXGNUQW6MeYCY8xOY8wuY8y3hjn+340x74Ve7xtjAsaYvOGuddzVboDMqZA7Y8DutZv24klJ5Py5U6NUMBGRsXXEQDfGJAI/Az4PzAGuNsbMiTzHWnu/tfZUa+2pwLeBN621jeNR4KNW87bTfm76+5h39wZ4ads+Pj+vUEP9RcQ1RlNDXwrsstbuttb6gKeBlYc5/2rgN2NRuE+stR6aa4YMKPrz9gO09fg1s6KIuMpoAr0YqI3YrgvtG8IY4wEuAJ4d4fj1xphKY0xlQ0PD0Zb16IXbzwcG+trNe5mWncbpJ+SPfxlERI6T0QT6cOPh7QjnXgT850jNLdbaNdbaJdbaJZMnTx5tGY9d7QZI9sC0BeFdh9p7ePOjBlYuLNJQfxFxldEEeh0wPWK7BKgf4dyrmCjNLeC0nxcvhsTk8K51W+oJBC2XLSyJYsFERMbeaAL9XWCmMWaGMSYFJ7RfGHySMSYHOAd4fmyLeIx62mH/+8M2t8wtyuaUaVlRKpiIyPg4YqBba/3ATcDLwA7gt9baD4wxNxhjbog49VLgT9bajvEp6lHaWwk2MCDQdx1sY2tdC5eq77mIuNCo+uxZa18CXhq075FB248Dj49VwT6xmncAAyWnhXc9Fxrqf7GG+ouIC7l3pGjNOzB1HqTlhHf9afsBzjypgClZGuovIu7jzkAP+KHu3QHD/f2BINXeDuYV5xzmjSIiscudgX7wA/C1D3igxb6WbnoDlrI8TxQLJiIyftwZ6MMMKKryOvdqy/IzolEiEZFx595Azy6GSf3d56u8nQCUF6iGLiLu5L5At9YJ9EH9z2u8HaQmJTBVN0RFxKXcF+gttdBWP+SB0FXeTsryPSRouL+IuJT7Ar1mg7McVEOv9nao/VxEXM2Fgf42pGTB1LnhXcGgpdrbSXm+2s9FxL3cF+i1G6BkCSQkhncdaOumxx9UDV1EXM1dgd7VDAc+GPJA6KpDTg+XMtXQRcTF3BXodZWAHfJA6OpQH/Ry1dBFxMXcFeg1b4NJhOIlA3ZXeTtJTjQU5qjLooi4l7sCvXYDTJsPqZkDdld7O5ie6yEp0V0/rohIJPckXKDXaXIZ1H4OUB3qgy4i4mbuCfR9W8HfNaT93FqrPugiEhfcE+i1oQm5Bo0QPdTuo8MXUB90EXE99wR6zdswqQyyCwfs7uvhUlagGrqIuJs7At1aZ8j/oOH+EDHLoppcRMTl3BHojbuh4+CwgV7t7SDBQPGk9CgUTETk+HFHoNeGJuSaPnwNvTg3nZQkd/yoIiIjcUfK1bzjPAx68qwhh6q9HWpuEZG44J5An74MEob+OOqDLiLxIvYDvbMRDu0ctv28udNHS1evaugiEhdiP9CP0H4OejC0iMSH2A/0mncgIRmKFw051D/LoppcRMT93BHoRadC8tBuiVWHOjEGpucp0EXE/WI70Hu7oX6Tc0N0GNXeDqZlp5GWnDjscRERN4ntQN/3HgR8w86wCFDl7VAPFxGJG7Ed6DV9E3KNVEPvVA8XEYkbsR3otRsg/yTInDzkUFt3L94On3q4iEjciN1AtzY0oGhod0VwauegHi4iEj9GFejGmAuMMTuNMbuMMd8a4Zzlxpj3jDEfGGPeHNtiDuPQx9DVOOSBFn2q1QddROJM0pFOMMYkAj8DPgvUAe8aY16w1m6POGcS8BBwgbW2xhgzZbwKHFbztrM8zA1RQDdFRSRujKaGvhTYZa3dba31AU8DKwed8xXgOWttDYC19uDYFnMYtRvAk++0oQ+j2tvB5KxUMlKP+G+WiIgrjCbQi4HaiO260L5IJwO5xpg3jDEbjTH/ZbgLGWOuN8ZUGmMqGxoajq3EfWredtrPjRn2cJW3kzINKBKRODKaQB8uMe2g7SRgMfBF4HPAXcaYk4e8ydo11tol1tolkycP7Zkyau0HnYdajNB+DujB0CISd0bTHlEHTI/YLgHqhznnkLW2A+gwxqwHKoCPxqSUg/VNyDVC+3mnz8+B1h71cBGRuDKaGvq7wExjzAxjTApwFfDCoHOeB84yxiQZYzzAMmDH2BY1Qs07kJgKhRXDH24M9XDRg6FFJI4csYZurfUbY24CXgYSgUettR8YY24IHX/EWrvDGPNHYCsQBH5hrX1/3Epd8w4UL4ak1GEPqw+6iMSjUXUBsda+BLw0aN8jg7bvB+4fu6KNwNfpzOFyxs0jntI3bW5ZnmroIhI/Ym+kaP0mCPpHHCEKTg+XXE8yOZ7k41gwEZHoir1AByg/C6YvHfGweriISDyKvUAv/zRc+yJ48kY8peqQHgwtIvEn9gL9CHr8AepbulRDF5G447pAr23swlr1cBGR+OO6QA/3cFENXUTijAsDXX3QRSQ+uTDQO8hKTSIvIyXaRREROa5cF+hV3k7KCjyYEWZhFBFxK9cFuvqgi0i8clWg9waC1DV1qf1cROKSqwK9vrkLf9BqDhcRiUuuCvSq8IOhVUMXkfjjqkDv64NernnQRSQOuSzQO0lLTmBK1vDzpIuIuJnLAr2D8vwMdVkUkbjkqkCv8mqWRRGJX64J9EDQUuPtpFx90EUkTrkm0Pe3duMLBDWoSETilmsCvfpQ3yyLanIRkfjkmkBXH3QRiXeuCfRqbwcpiQkU5qRHuygiIlHhmkCv8nYwPS+dxAR1WRSR+OSaQK9WDxcRiXOuCHRrLdXeTvVwEZG45opAb2jroas3QHmBboiKSPxyRaD393BRDV1E4pdLAj3UBz1PNXQRiV+uCPRqbweJCYbiXHVZFJH45YpAr/J2UpKbTnKiK34cEZFj4ooE1IOhRURGGejGmAuMMTuNMbuMMd8a5vhyY0yLMea90Ot7Y1/U4fV1WdSDoUUk3iUd6QRjTCLwM+CzQB3wrjHmBWvt9kGnvmWtvXAcynhYTZ29tHX7VUMXkbg3mhr6UmCXtXa3tdYHPA2sHN9ijV5fDxfV0EUk3h2xhg4UA7UR23XAsmHO+5QxZgtQD/yLtfaDwScYY64HrgcoLS09+tIOo+/B0Kqhi3wyvb291NXV0d3dHe2iCJCWlkZJSQnJycmjfs9oAn242a7soO1NQJm1tt0Y8wXgd8DMIW+ydg2wBmDJkiWDr3FMqg51YgxMz1OXRZFPoq6ujqysLMrLy/Vc3iiz1uL1eqmrq2PGjBmjft9omlzqgOkR2yU4tfDID2+11raH1l8Cko0xBaMuxSdQ7e2gKCed1KTE4/FxIq7V3d1Nfn6+wnwCMMaQn59/1P9bGk2gvwvMNMbMMMakAFcBLwz68Gkm9FtgjFkauq73qEpyjPRgaJGxozCfOI7lz+KITS7WWr8x5ibgZSAReNRa+4Ex5obQ8UeAy4FvGmP8QBdwlbV2TJpUjqTa28EF8wqPx0eJiExoo2lD72tGeWnQvkci1h8EHhzboh1ZS1cvTZ296uEiIkKMjxSt0SyLInIM/H5/tIswLkZVQ5+own3QNQ+6yJj6/roP2F7fOqbXnFOUzd0XzT3ieZdccgm1tbV0d3dzyy23cP311/PHP/6R1atXEwgEKCgo4NVXX6W9vZ2bb76ZyspKjDHcfffdfOlLXyIzM5P29nYAnnnmGV588UUef/xxrr32WvLy8ti8eTOLFi3iyiuv5NZbb6Wrq4v09HQee+wxTjnlFAKBAHfeeScvv/wyxhiuu+465syZw4MPPsjatWsB+POf/8zDDz/Mc889N6bf0ScV04He1we9VNPmirjGo48+Sl5eHl1dXZx22mmsXLmS6667jvXr1zNjxgwaGxsB+MEPfkBOTg7btm0DoKmp6YjX/uijj3jllVdITEyktbWV9evXk5SUxCuvvMLq1at59tlnWbNmDXv27GHz5s0kJSXR2NhIbm4uN954Iw0NDUyePJnHHnuMVatWjev3cCxiOtCrvJ1MyUrFkxLTP4bIhMs0wqcAAAo/SURBVDOamvR4+elPfxquCdfW1rJmzRrOPvvscH/svLw8AF555RWefvrp8Ptyc3OPeO0rrriCxESni3NLSwtf+9rX+PjjjzHG0NvbG77uDTfcQFJS0oDPu+aaa/j1r3/NqlWrePvtt3niiSfG6CceOzGdhNXeDj0YWsRF3njjDV555RXefvttPB4Py5cvp6Kigp07dw4511o7bNe+yH2D+3FnZPTnxV133cVnPvMZ1q5dS1VVFcuXLz/sdVetWsVFF11EWloaV1xxRTjwJ5KYvimqPugi7tLS0kJubi4ej4cPP/yQd955h56eHt5880327NkDEG5yOf/883nwwf7OdX1NLlOnTmXHjh0Eg8FwTX+kzyouLgbg8ccfD+8///zzeeSRR8I3Tvs+r6ioiKKiIn74wx9y7bXXjtnPPJZiNtA7evw0tPVQXqAauohbXHDBBfj9fhYsWMBdd93F6aefzuTJk1mzZg2XXXYZFRUVXHnllQB897vfpampiXnz5lFRUcHrr78OwH333ceFF17IueeeS2HhyGNU7rjjDr797W9z5plnEggEwvu/8Y1vUFpayoIFC6ioqOCpp54KH/vqV7/K9OnTmTNnzjh9A5+MOU7jf4ZYsmSJraysPOb3b69v5Qs/fYsHv7KQCxcUjWHJROLTjh07mD17drSLMaHddNNNLFy4kK9//evH5fOG+zMxxmy01i4Z7vyJ1wg0SjWNfdPmqoYuIuNv8eLFZGRk8OMf/zjaRRlRzAZ6VWhQUana0EXkONi4cWO0i3BEMduGXu3tID8jhey00c8VLCLiZjEb6FWH1MNFRCRSzAZ6tbdDc7iIiESIyUDv7g1Q39KtGrqISISYDPTaRueGqHq4iIj0i8lArwpPm6sauki8yszMjHYRJpyY7LbYN8uiaugi4+QP34L928b2mtPmw+fvG9trTgB+v3/CzOsSkzX0am8n2WlJTPKoy6KIW9x555089NBD4e177rmH73//+6xYsYJFixYxf/58nn/++VFdq729fcT3PfHEE+Fh/ddccw0ABw4c4NJLL6WiooKKigr++te/UlVVxbx588Lv+9GPfsQ999wDwPLly1m9ejXnnHMODzzwAOvWrWPZsmUsXLiQ8847jwMHDoTLsWrVKubPn8+CBQt49tln+eUvf8ltt90Wvu7Pf/5zbr/99mP+3gaw1kbltXjxYnus/uEX79iL/v2tY36/iAy1ffv2qH7+pk2b7Nlnnx3enj17tq2urrYtLS3WWmsbGhrsiSeeaIPBoLXW2oyMjBGv1dvbO+z73n//fXvyySfbhoYGa621Xq/XWmvtl7/8ZfuTn/zEWmut3++3zc3Nds+ePXbu3Lnha95///327rvvttZae84559hvfvOb4WONjY3hcv385z+3t99+u7XW2jvuuMPecsstA85rb2+3J5xwgvX5fNZaaz/1qU/ZrVu3DvtzDPdnAlTaEXJ1Yvw/4ShVezupmD4p2sUQkTG0cOFCDh48SH19PQ0NDeTm5lJYWMhtt93G+vXrSUhIYO/evRw4cIBp06Yd9lrWWlavXj3kfa+99hqXX345BQUFQP9c56+99lp4fvPExERycnKO+MCMvknCAOrq6rjyyivZt28fPp8vPHf7SHO2n3vuubz44ovMnj2b3t5e5s+ff5Tf1vBiLtB9/iB1TZ1cXKEJuUTc5vLLL+eZZ55h//79XHXVVTz55JM0NDSwceNGkpOTKS8vHzLH+XBGep8dYa7z4SQlJREMBsPbh5tb/eabb+b222/n4osv5o033gg3zYz0ed/4xje49957mTVr1pg++Sjm2tD3NncRtOrhIuJGV111FU8//TTPPPMMl19+OS0tLUyZMoXk5GRef/11qqurR3Wdkd63YsUKfvvb3+L1eoH+uc5XrFjBww8/DEAgEKC1tZWpU6dy8OBBvF4vPT09vPjii4f9vL651X/1q1+F9480Z/uyZcuora3lqaee4uqrrx7t13NEMRfo/Q+GVg8XEbeZO3cubW1tFBcXU1hYyFe/+lUqKytZsmQJTz75JLNmzRrVdUZ639y5c/nOd77DOeecQ0VFRfhm5AMPPMDrr7/O/PnzWbx4MR988AHJycl873vfY9myZVx44YWH/ex77rmHK664grPOOivcnAMjz9kO8OUvf5kzzzxzVI/OG62Ymw+9sqqRNet3c+9l8ynITB2HkonEJ82HfnxdeOGF3HbbbaxYsWLEc1w/H/qS8jyWlOdFuxgiIsekubmZpUuXUlFRcdgwPxYxF+giIn22bdsW7kveJzU1lQ0bNkSpREc2adIkPvroo3G5tgJdRMKOphfIRDB//nzee++9aBdjXBxLc3jM3RQVkfGRlpaG1+s9piCRsWWtxev1kpaWdlTvUw1dRAAoKSmhrq6OhoaGaBdFcP6BLSkpOar3KNBFBIDk5OTwCEeJTWpyERFxCQW6iIhLKNBFRFwiaiNFjTENwOgmZhiqADg0hsVxI31Hh6fv58j0HR1etL6fMmvt5OEORC3QPwljTOVIQ1/Foe/o8PT9HJm+o8ObiN+PmlxERFxCgS4i4hKxGuhrol2AGKDv6PD0/RyZvqPDm3DfT0y2oYuIyFCxWkMXEZFBFOgiIi4Rc4FujLnAGLPTGLPLGPOtaJdnIjLGVBljthlj3jPGHP1joVzGGPOoMeagMeb9iH15xpg/G2M+Di3H7jlgMWiE7+geY8ze0O/Re8aYL0SzjNFkjJlujHndGLPDGPOBMeaW0P4J9XsUU4FujEkEfgZ8HpgDXG2MmRPdUk1Yn7HWnjrR+slGyePABYP2fQt41Vo7E3g1tB3PHmfodwTwk9Dv0anW2peOc5kmEj/wz9ba2cDpwI2h7JlQv0cxFejAUmCXtXa3tdYHPA2sjHKZZIKz1q4HGgftXgn0PZ79V8Alx7VQE8wI35GEWGv3WWs3hdbbgB1AMRPs9yjWAr0YqI3Yrgvtk4Es8CdjzEZjzPXRLswENdVauw+cv6zAlCiXZ6K6yRizNdQkE9fNUn2MMeXAQmADE+z3KNYCfbhnY6nf5VBnWmsX4TRN3WiMOTvaBZKY9DBwInAqsA/4cXSLE33GmEzgWeBWa21rtMszWKwFeh0wPWK7BKiPUlkmLGttfWh5EFiL01QlAx0wxhQChJYHo1yeCcdae8BaG7DWBoGfE+e/R8aYZJwwf9Ja+1xo94T6PYq1QH8XmGmMmWGMSQGuAl6IcpkmFGNMhjEmq28dOB94//DviksvAF8LrX8NeD6KZZmQ+oIq5FLi+PfIOE/O/iWww1r7vyMOTajfo5gbKRrqOvV/gETgUWvtv0a5SBOKMeYEnFo5OI8YfCrevyNjzG+A5TjTnR4A7gZ+B/wWKAVqgCustXF7U3CE72g5TnOLBaqAf+prL443xphPA28B24BgaPdqnHb0CfN7FHOBLiIiw4u1JhcRERmBAl1ExCUU6CIiLqFAFxFxCQW6iIhLKNBFRFxCgS4i4hL/HzgtGxezvAIwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tf.random.set_seed(33)\n",
    "\n",
    "MODEL_DIR = os.path.join(LOGDIR, 'dnn')\n",
    "shutil.rmtree(MODEL_DIR, ignore_errors=True)\n",
    "\n",
    "BATCH_SIZE = 300\n",
    "EPOCHS = 100\n",
    "EMBED_DIM = 10\n",
    "PATIENCE = 0\n",
    "\n",
    "dnn_model = build_dnn_model(embed_dim=EMBED_DIM)\n",
    "\n",
    "dnn_history = dnn_model.fit(\n",
    "    X_train, Y_train,\n",
    "    epochs=EPOCHS,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    validation_data=(X_valid, Y_valid),\n",
    "    callbacks=[callbacks.EarlyStopping(patience=PATIENCE),\n",
    "               callbacks.TensorBoard(MODEL_DIR)],\n",
    ")\n",
    "\n",
    "pd.DataFrame(dnn_history.history)[['loss', 'val_loss']].plot()\n",
    "pd.DataFrame(dnn_history.history)[['accuracy', 'val_accuracy']].plot()\n",
    "\n",
    "dnn_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transfer Learning with Pre-trained Embedding\n",
    "\n",
    "We can also use a word embedding from a pre-trained modle using a [Neural Probabilistic Language Model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf). TF-Hub has a 50-dimensional one called \n",
    "[nnlm-en-dim50-with-normalization](https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1), which also normalizes the vectors produced. \n",
    "\n",
    "Once loaded from its url, the TF-hub module can be used as a normal Keras layer in a sequential or functional model. Since we have enough data to fine-tune the parameters of the pre-trained embedding itself, we will set `trainable=True` in the `KerasLayer` that loads the pre-trained embedding:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "NNLM = \"https://tfhub.dev/google/nnlm-en-dim50/2\"\n",
    "\n",
    "nnlm_module = KerasLayer(\n",
    "    handle=NNLM,\n",
    "    output_shape=[50],\n",
    "    input_shape=[],\n",
    "    dtype=tf.string,\n",
    "    trainable=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With this module, we do not need to pad our inputs. The NNLM module returns a 50-dimensional vector given a word or sentence. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 50), dtype=float32, numpy=\n",
       "array([[ 0.0958989 , -0.34283403, -0.0492865 , -0.09070477,  0.15877678,\n",
       "        -0.2123544 ,  0.28213888, -0.02812295, -0.07855891, -0.13102815,\n",
       "         0.11162009,  0.00899508,  0.01711924,  0.3225362 , -0.13289747,\n",
       "         0.11935385,  0.04386025,  0.06534779,  0.2200468 , -0.13539287,\n",
       "        -0.0296053 , -0.06080014,  0.12862371,  0.23304917, -0.04424817,\n",
       "         0.07436227, -0.1898077 , -0.13270935,  0.21959059,  0.10597934,\n",
       "         0.03580458,  0.14275002, -0.06624421, -0.3247055 ,  0.04618761,\n",
       "        -0.11603005,  0.06651008,  0.10887001, -0.05413236, -0.07126983,\n",
       "         0.02225055,  0.2645486 , -0.04697315,  0.06729111, -0.14438024,\n",
       "         0.06355232, -0.05749882, -0.04587579,  0.23790349,  0.258379  ],\n",
       "       [ 0.11347695, -0.04064287,  0.1053718 , -0.2368139 , -0.08755025,\n",
       "        -0.29770336, -0.00098698,  0.2312349 , -0.05596383,  0.04687293,\n",
       "         0.07230621, -0.10018747,  0.17597003, -0.04471372, -0.16409421,\n",
       "        -0.21718104, -0.13013352,  0.10073684,  0.04346658, -0.121227  ,\n",
       "        -0.05811132, -0.17935495,  0.08445173, -0.18978111,  0.14924097,\n",
       "         0.1283434 , -0.14006372,  0.10938524,  0.00990795, -0.31790745,\n",
       "        -0.04520534, -0.02819821,  0.13765122, -0.06532548, -0.18180048,\n",
       "         0.03769377, -0.12396756, -0.10564797,  0.01820223, -0.22254522,\n",
       "         0.06358314, -0.25590476,  0.25336912,  0.2768555 , -0.09150941,\n",
       "        -0.1493839 , -0.08984404,  0.13982305,  0.4764217 , -0.17479022]],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nnlm_module(tf.constant([\"holy cash cow  batman - content is back\",\n",
    "                         \"close look at a flu outbreak upends some common wisdom\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With this in mind, we can simplify our data inputs since do not need to integerize or pad. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, Y_train = titles_train.values, encode_labels(sources_train)\n",
    "X_valid, Y_valid = titles_valid.values, encode_labels(sources_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['holy cash cow  batman - content is back',\n",
       "       'show hn  a simple and configurable deployment tool for github projects',\n",
       "       'show hn  neural turing machine in pure numpy. implements all 5 tasks from paper'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build DNN model using TF-Hub Embedding layer\n",
    "\n",
    "Next, we can add this TF-Hub module to our DNN model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_hub_model():\n",
    "    model = models.Sequential([\n",
    "        KerasLayer(handle=NNLM,\n",
    "                   output_shape=[50],\n",
    "                   input_shape=[],\n",
    "                   dtype=tf.string,\n",
    "                   trainable=True),\n",
    "        layers.Dense(N_CLASSES, activation='softmax')\n",
    "    ])\n",
    "\n",
    "    model.compile(\n",
    "        optimizer='adam',\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 76962 samples, validate on 19241 samples\n",
      "Epoch 1/100\n",
      "76962/76962 [==============================] - 13s 168us/sample - loss: 0.7125 - accuracy: 0.7258 - val_loss: 0.4898 - val_accuracy: 0.8121\n",
      "Epoch 2/100\n",
      "76962/76962 [==============================] - 12s 156us/sample - loss: 0.4067 - accuracy: 0.8485 - val_loss: 0.4012 - val_accuracy: 0.8404\n",
      "Epoch 3/100\n",
      "76962/76962 [==============================] - 12s 156us/sample - loss: 0.3231 - accuracy: 0.8793 - val_loss: 0.3818 - val_accuracy: 0.8435\n",
      "Epoch 4/100\n",
      "76962/76962 [==============================] - 12s 156us/sample - loss: 0.2763 - accuracy: 0.8979 - val_loss: 0.3825 - val_accuracy: 0.8445\n",
      "Epoch 5/100\n",
      "76962/76962 [==============================] - 12s 156us/sample - loss: 0.2437 - accuracy: 0.9107 - val_loss: 0.3925 - val_accuracy: 0.8428\n",
      "Epoch 6/100\n",
      "76962/76962 [==============================] - 12s 154us/sample - loss: 0.2194 - accuracy: 0.9205 - val_loss: 0.4084 - val_accuracy: 0.8400\n",
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_layer_4 (KerasLayer)   (None, 50)                48190600  \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 3)                 153       \n",
      "=================================================================\n",
      "Total params: 48,190,753\n",
      "Trainable params: 48,190,753\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "CPU times: user 57.9 s, sys: 13.4 s, total: 1min 11s\n",
      "Wall time: 1min 14s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXyU5bn/8c+dfU8gJJCNhACyQ9AIooJsCu51aQUXrFUpWkDbatXT5djFY1tPWzcqP+tWT61iXVoXBFFQQFFJMKwBZCdhSVgSEiBku39/zASTkGUCM5nM5Pt+vebFLM88c42SL1eu555njLUWERHxfQHeLkBERNxDgS4i4icU6CIifkKBLiLiJxToIiJ+IshbL9ytWzebkZHhrZcXEfFJubm5B6y1CU095rVAz8jIICcnx1svLyLik4wxO5t7TCMXERE/oUAXEfETCnQRET/htRm6iHROVVVVFBQUUFFR4e1SOrSwsDBSU1MJDg52+TkKdBFpVwUFBURHR5ORkYExxtvldEjWWg4ePEhBQQG9evVy+XkauYhIu6qoqCA+Pl5h3gJjDPHx8W3+LUaBLiLtTmHeutP5b+Rzgb7jwFF+/e56qmpqvV2KiEiH4nOBvu1AOS9+toO3VxV6uxQR8VFRUVHeLsEjfC7Qx/VLZGhqLE8v2aIuXUSkHpcC3Rgz2RizyRizxRjzYBOP32+MyXNe1hljaowxXd1frmOuNHt8X3YdOsbbX6tLF5HTZ63l/vvvZ/DgwQwZMoR58+YBsHfvXsaMGUNWVhaDBw9m2bJl1NTU8P3vf//ktn/5y1+8XP2pWl22aIwJBOYAFwMFwEpjzDvW2g1121hrHwMec25/JfBja+0hz5QMEwYkMjglhjlLtnDt8BSCAn3uFw0RAX797no27Dni1n0OTI7hv68c5NK2b731Fnl5eaxevZoDBw5w7rnnMmbMGP75z38yadIkfv7zn1NTU8OxY8fIy8ujsLCQdevWAVBSUuLWut3BlSQcAWyx1m6z1lYCrwFXt7D9VOBVdxTXnLoufefBY/w7b48nX0pE/Njy5cuZOnUqgYGBdO/enYsuuoiVK1dy7rnn8uKLL/Lwww+zdu1aoqOjyczMZNu2bcyaNYsFCxYQExPj7fJP4coHi1KA3fVuFwAjm9rQGBMBTAZmNvP4dGA6QM+ePdtUaGMXD+zOwKQYnl78Dd/JSlaXLuKDXO2kPcVa2+T9Y8aMYenSpbz//vvccsst3H///UybNo3Vq1ezcOFC5syZw+uvv84LL7zQzhW3zJUUbGoxZNP/FeBK4LPmxi3W2mettdnW2uyEhCZP5+syYwyzJ/Rlx8FjvLNaXbqItN2YMWOYN28eNTU1FBcXs3TpUkaMGMHOnTtJTEzkzjvv5Pbbb2fVqlUcOHCA2tparrvuOn7729+yatUqb5d/Clc69AIgrd7tVKC5BJ2Ch8ct9V0ysDv9e0Tz9OItXJ2VQmCAPqwgIq675pprWLFiBcOGDcMYwx//+Ed69OjB3//+dx577DGCg4OJiori5ZdfprCwkNtuu43aWsfqukcffdTL1Z/KNPcrx8kNjAkCNgMTgEJgJXCjtXZ9o+1ige1AmrX2aGsvnJ2dbd3xBRcL1u1lxj9W8fgNWXxneMoZ709EPCs/P58BAwZ4uwyf0NR/K2NMrrU2u6ntWx25WGurcczEFwL5wOvW2vXGmBnGmBn1Nr0G+NCVMHenSwb2oH+PaJ5c/A01tS3/4yQi4s9cOpJorZ1vrT3LWtvbWvuI87651tq59bZ5yVo7xVOFNicgwDFL31Z8lPfWaJYuIp2XXywNmTyoB/26R/Pkx+rSRaTz8otADwgwzJrQh63FR3l/7V5vlyMi4hV+EegAlw1Oom9iFE99/A216tJFpBPym0B3dOl9+aaonPnr1KWLSOfjN4EOcPmQJHonRPKkunQR6YT8KtADnSteNu8vZ8H6fd4uR0T8QEvnTt+xYweDBw9ux2pa5leBDnDF0GR16SLSKbny0X+fEhhgmDW+L/fOy2Ph+n1cOiTJ2yWJSHM+eBD2rXXvPnsMgUt/3+zDDzzwAOnp6dx9990APPzwwxhjWLp0KYcPH6aqqorf/e53XH11SyeVPVVFRQV33XUXOTk5BAUF8ec//5lx48axfv16brvtNiorK6mtreXNN98kOTmZ733vexQUFFBTU8Mvf/lLbrjhhjN62+CHHTrAlcOSyewWyRPq0kWkkSlTppz8IguA119/ndtuu423336bVatWsWTJEn760582eybG5syZMweAtWvX8uqrr3LrrbdSUVHB3Llzueeee8jLyyMnJ4fU1FQWLFhAcnIyq1evZt26dUyePNkt783vOnRwdOkzx/fhJ6+v5sMN+5k8uIe3SxKRprTQSXvK8OHDKSoqYs+ePRQXF9OlSxeSkpL48Y9/zNKlSwkICKCwsJD9+/fTo4fr2bF8+XJmzZoFQP/+/UlPT2fz5s2MGjWKRx55hIKCAq699lr69u3LkCFDuO+++3jggQe44oorGD16tFvem1926ABXDUsmIz6CJz/+ps3/0oqIf7v++ut54403mDdvHlOmTOGVV16huLiY3Nxc8vLy6N69OxUVFW3aZ3M5c+ONN/LOO+8QHh7OpEmTWLx4MWeddRa5ubkMGTKEhx56iN/85jfueFv+G+hBgQHMHN+XDXuPsGjDfm+XIyIdyJQpU3jttdd44403uP766yktLSUxMZHg4GCWLFnCzp0727zPMWPG8MorrwCwefNmdu3aRb9+/di2bRuZmZnMnj2bq666ijVr1rBnzx4iIiK4+eabue+++9x2bnW/DXSA72Qlkx4fwRPq0kWknkGDBlFWVkZKSgpJSUncdNNN5OTkkJ2dzSuvvEL//v3bvM+7776bmpoahgwZwg033MBLL71EaGgo8+bNY/DgwWRlZbFx40amTZvG2rVrGTFiBFlZWTzyyCP84he/cMv7avV86J7irvOht+ZfObu5/401PDctm4kDu3v89USkZTofuuvcfj50X3fN8BR6dlWXLiL+z+8DPSgwgJnj+rC2sJTFG4u8XY6I+KC1a9eSlZXV4DJy5Ehvl3UKv1y22Ng1Z6fw1JJveOLjbxjfPxFj9N2jIt5krfWpn8MhQ4aQl5fXrq95OhMFv+/QAYIDA/jR2D6sKSjlk03F3i5HpFMLCwvj4MGDGoG2wFrLwYMHCQsLa9PzOkWHDnDt2ak8tXgLj3/8DWP7JfhUdyDiT1JTUykoKKC4WM1VS8LCwkhNTW3TczpNoIcEBfCjcX34r7fX8unmYsb2S/R2SSKdUnBwML169fJ2GX6pU4xc6lx/TiopceFa8SIifqlTBXpIUAB3j+vN17tKWPbNAW+XIyLiVp0q0AG+e04aybFh6tJFxO90ukAPCQrgrnF9yN15mM+2HPR2OSIibtPpAh3ge9mpJMWG8fhHm9Wli4jf6JSBHhoUyF1je5Oz8zCfb1WXLiL+oVMGOsD3stPoHhPKEx9pli4i/qHTBnpYcCB3XdSbr3YcYsU2deki4vs6baADTBnRk8RoR5cuIuLrOnWghwU7Zulfbj/EF+rSRcTHdepAB5g6oicJ6tJFxA+4FOjGmMnGmE3GmC3GmAeb2WasMSbPGLPeGPOpe8v0nLDgQGZc1JsV2w7ypbp0EfFhrQa6MSYQmANcCgwEphpjBjbaJg74K3CVtXYQ8F0P1OoxN43sSbeoUJ74WF26iPguVzr0EcAWa+02a20l8BpwdaNtbgTestbuArDW+tRXAzm69Ew+33qQlTsOebscEZHT4kqgpwC7690ucN5X31lAF2PMJ8aYXGPMtKZ2ZIyZbozJMcbkdLRzId80Mp1uUSGapYuIz3Il0Jv6JojGn8QJAs4BLgcmAb80xpx1ypOsfdZam22tzU5ISGhzsZ4UHhLI9DGZLN9ygNyd6tJFxPe4EugFQFq926nAnia2WWCtPWqtPQAsBYa5p8T2c/N56cRHhvC4unQR8UGuBPpKoK8xppcxJgSYArzTaJv/AKONMUHGmAhgJJDv3lI9LyIkiOljMln2zQFW7Trs7XJERNqk1UC31lYDM4GFOEL6dWvtemPMDGPMDOc2+cACYA3wFfCctXad58r2nFtGpdM1UrN0EfE9Ln2nqLV2PjC/0X1zG91+DHjMfaV5R0RIEHeOzuQPCzby9a7DDO/ZxdsliYi4pNN/UrQp00al0yUiWOvSRcSnKNCbEBkaxB2jM/lkUzF5u0u8XY6IiEsU6M249fwM4iKCeVJduoj4CAV6M6JCg7jjwl4s3ljEmgJ16SLS8SnQW3Dr+RnEhqtLFxHfoEBvQXRYMHdc2IuP8otYV1jq7XJERFqkQG/FrRdkEBMWpBUvItLhKdBbERMWzO0XZrJow3516SLSoSnQXfD9CzKIDgvSLF1EOjQFugtiw4P5wQW9+HDDfjbsOeLtckREmqRAd9EPLuhFdKi6dBHpuBToLoqNCOa2CzJYsH4f+XvVpYtIx6NAb4MfXOjo0p9arC5dRDoeBXobxEWE8P0LMpi/dh+b9pV5uxwRkQYU6G10+4W9iAoN4kl16SLSwSjQ2yguIoRbz09n/tq9bN6vLl1EOg4F+mm448JMIoIDteJFRDoUBfpp6BIZwrTzM3h/7V6+UZcuIh2EAv003Tk6k/DgQJ5avMXbpYiIAAr009Y1MoRbRqXz7po9bCkq93Y5IiIK9DMxfXQmYUGBPK0VLyLSASjQz0B8VCjTRqXzzuo9bCtWly4i3qVAP0N3jskkNCiQpzVLFxEvU6CfoW5Rodx8Xk/+nVfI9gNHvV2OiHRiCnQ3mD6mNyFBATrHi4h4lQLdDRKiQ7lpZDr/ydvDDnXpIuIlCnQ3+eFFmQQFGJ5eolm6iHiHAt1NEqPDuGlkOm9/XcjOg+rSRaT9KdDdaIazS5+jLl1EvECB7kaJMWFMHdGTt1YVsvvQMW+XIyKdjALdze4a25sAdeki4gUuBboxZrIxZpMxZosx5sEmHh9rjCk1xuQ5L79yf6m+oXtMGFPPTeON3AJ16SLSrloNdGNMIDAHuBQYCEw1xgxsYtNl1tos5+U3bq7Tp8wY25sAY/jrJ1u9XYqIdCKudOgjgC3W2m3W2krgNeBqz5bl25Jiw7nh3DT+lbObgsPq0kWkfbgS6CnA7nq3C5z3NTbKGLPaGPOBMWZQUzsyxkw3xuQYY3KKi4tPo1zfcdfY3hiDunQRaTeuBLpp4j7b6PYqIN1aOwx4Cvh3Uzuy1j5rrc221mYnJCS0rVIfkxz3bZdeWHLc2+WISCfgSqAXAGn1bqcCe+pvYK09Yq0td16fDwQbY7q5rUofddfYPgA884lWvIiI57kS6CuBvsaYXsaYEGAK8E79DYwxPYwxxnl9hHO/B91dLADVlbB5IdjGvyR0PClx4Xw3O43XVxawt1Rduoh4VquBbq2tBmYCC4F84HVr7XpjzAxjzAznZtcD64wxq4EngSnWeihxV78K//we/N93oGijR17Cne4e2xuL5RnN0kXEw4yncrc12dnZNicnp+1PrKmGnBdgySNwogxG3AljH4TwLu4v0k0eemsNb+YWsvRn4+gRG+btckTEhxljcq212U095nufFA0MgpHTYdYqOOdW+OpZeOocyHkRamu8XV2T7h7bh1prmfupunQR8RzfC/Q6kfFwxV9g+qeQ0B/euxeeHQs7V3i7slOkdY3gurNT+edXu9h/pMLb5YiIn/LdQK+TNBS+/z5c/wIcOwgvToY3bofSQm9X1sCPxvWhplazdBHxHN8PdABjYPB1MHMljPkZ5L8LT2fD0segqmN0xD3jI7ju7BRe/WoXRerSRcQD/CPQ64REwvifw8yvoM8EWPw7mDMC8t/rEMscZ47rS3WtZe6n27xdioj4If8K9DpdMuCGf8C0/0BwBMy7qUMsc+wZH8E1w1N45cudFJWpSxcR9/LPQK+TORZmLIdL/wh7voZnzocPHoTjJV4raea4PlTXWp5Vly4ibubfgQ7OZY4/hFlfw9nT4Mu58NTZkPuSV5Y5ZnSL5OqsZP7x5U6Ky060++uLiP/y/0CvExkPVz4OP/wUup0F794DfxsHu75o91Jmje9LZXUtf1umLl1E3KfzBHqdpGFw2wdw3fNQXgwvTII374Aje1p/rpv06hbJ1VkpvLxiBwfK1aWLiHt0vkAHxzLHIdfDrBwYcz9seAeeyoal/9tuyxxnju/j6NKXqksXEffonIFeJyQSxv/Cscyx9zhY/Fv460jY+L7Hlzn2TojiqmHJvLxiJwfVpYuIG3TuQK/TJQOmvAK3/BuCwuC1G+Ef10LxJo++7MzxfamoruFvy7Z79HVEpHNQoNfXe5xjmePkP0BhrmOZ44KHPLbMsU9iFFcOTeblFTs4dLTSI68hIp2HAr2xwGA4b4bjbI7Db4YvnnGczTH37x5Z5jh7Qh+OV9XwnFa8iMgZUqA3J7IbXPkETP8E4vvAu7Phb+Nh15dufZk+idFcPiSJv3++g8Pq0kXkDCjQW5OcBT9Y4FzmWAQvXAJv3unWZY6zJ/TlWFUNzy/XLF1ETp8C3RX1lzmOvg82/MexzHHZn6D6zFeonNU9mssGJ/HS5zsoOaYuXUROjwK9LUIiYcIv4UdfOg6gfvwbmDMSNs4/42WOsyb0ofxEtbp0ETltCvTT0bWXc5nj2xAYAq9NPeNljv17xHDZkB689NkOSo9VubFYEeksFOhnovd4uOszmPx7KKhb5vhfUFF6WrubPaEvZSeqef4zdeki0nYK9DMVGAzn3QWzV0HWTfDFXx3LHFe9DLW1bdpV/x4xTB7Ugxc/207pcXXpItI2CnR3iewGVz3pWObYNRPemQXPjYfdX7VpN7Mn9KWsopoX1aWLSBsp0N0tOQt+sBCufQ7K9sHzF8NbP4Qje116+sDkGC4Z2J0Xlm/nSIW6dBFxnQLdE4yBod+FmTkw+qew/i3HGGbZn11a5jh7Ql+OVFTz0mc7PF+riPgNBbonhUbBhF85ljlmjoWPf+1Y5rjpgxaXOQ5OiWXigO48t2ybunQRcZkCvT10zYSp/4Sb33Isc3x1CrxyPRRvbvYp9050dOl/V5cuIi5SoLenPhMcyxwnPeo4WPrMKFj48yaXOTq69ESeW76dMnXpIuICBXp7CwyGUXc7zuaYdSOsmONc5vh/pyxzvGfCWZQer+LlFTu9VKyI+BIFurdEJcBVT8H0JdClF7wzE56bALtXntxkSGos4/sn8rdl2yg/Ue3FYkXEFyjQvS15ONz+IVz7NyjbC89PhLdnOJY8AvdM6EvJsSpeXrHDq2WKyGmoPAYHt8KO5bDmX/DZE44vzdn4vkdeLsiVjYwxk4EngEDgOWvt75vZ7lzgC+AGa+0bbqvS3xkDQ78H/S5znMFxxdOQ/y6MuY9h593N2H4JPPPJVrpHh3HN8BQCAoy3Kxbp3Gpr4Gix4zTaZXsdlyN1f+5xNGRle5o+DUhwJIR3gf6Xu70sY1s5S6AxJhDYDFwMFAArganW2g1NbLcIqABeaC3Qs7OzbU5OzhmU7scOboUPfwGb5kPXTPad/zDTv4hnTUEpg5Jj+K/LBnBBn27erlLEP1UcaRTSe74N67r7yveDbfQNZiYQorpDTBJEOy8xSRCdDNE9ICbZcV9YzBmVZ4zJtdZmN/mYC4E+CnjYWjvJefshAGvto422uxeoAs4F3lOgu8GWjxy/nh3YjE0bxbbg3ry/K5h1x+Lont6PaZPH0Dc9xdtViviGmipHENeFdNm+bzvsk131XqgsP/W5YbH1QtoZzPVDOiYZIhMgINDjb6OlQHdl5JIC7K53uwAY2egFUoBrgPE4Al3coc9EuOsi+OpZTN4/6b3/38yuLocQYC/wIhwLjCY4vhfB8ekQ57x0qbue5jiHu4g/sxaOH2559HFkr2NEQqMGNiD42066+0DHz9wpXXUPn/k5ciXQmxrYNm7rHwcesNbWGNP8fNcYMx2YDtCzZ09Xa+zcAoNh1I8cl7q/uId3UL5/K1+s+pr9OzeRtv8Ag8rW0rVqEaa6ouHzIxOc4d6zXtD3hC4ZEJsKQaFeeVsiLqmqgPJ9zY8+6jrtxn/vAcK7fttBJw1rosNOgoh4CPCftSFuGbkYY7bzbfB3A44B0621/25uvxq5uMeug8f448KNvLdmLwmRwTwwJp7vZFQRVLobSnY6Lod3QskuKN0NtfWXPxrHX+oGQV/vekwKBLp03FzEdbU1joOFFSVwvKThwcX6o48je+D4oVOfHxRWL5h7nBrSMUkQ1QOCw9r/vbWDM52hB+E4KDoBKMRxUPRGa+36ZrZ/Cc3Q213e7hL+5/18vtpxiN4JkTx46QAmDkikwW9MtTWOH5KSXQ2Dvu76kUIa/PJlAiE2pdEYp+56T8cPjR91N9IG1ZWOQK4odYRyXThXNLp+vKReeDv/PHGkmZ0ax2+UdSOP5g4uhndxrAzrpM4o0J07uAzHWCUQxwqWR4wxMwCstXMbbfsSCnSvsNayaMN+fr9gI9uKjzKiV1d+ftkAhqXFubaD6ko4UnBq0NddL9/fcPvAUMecPq5nw6CPy3Bcj4jv1D94HZq1UHW8hVBu5b6qYy3vPygcwuMgLM75Z2y9643uqwvxqO6OEaO06IwD3RMU6J5TVVPLayt388RHmzlQXsmVw5L52aR+pHWNOMMdH4cS5yjn8I5TQ7/xr8fBkfXGOI1DP93xQy2nz1o4UXb6oVxT2fL+Q6JdD+XG9+nYjMco0Dup8hPV/L9Pt/K3ZduorYVpo9KZOb4PcREhnnnBiiOOOf1h5+y+ZNe31w/vhMqyhtuHxX4b7g3GOs5/BNqyssBa56UWcF6nhftw3l/39/+U+5rarqn90sbXaryPRvfVdc0uhXLpqWuhGzCO/8YNwjbOhfu6QGiMjp90UAr0Tm5faQV/XrSJf+UWEBMWzMxxfZh2fjqhQZ5fM3tS3QqdxkF/8vouqD7e8DnBzt8oWgtZfxYQ1EQH7Eooxzk6bB3j8DsKdAFg474jPDp/I59uLia1Szj3T+rHlUOTO8apBKx1rHY4GfQ74dgh5wzeNPzTBJx6H877T16nifta2Ydxhl+bX6v+9m15rfr3OesNCmsYysEROg4hDSjQpYHl3xzgf+bns2HvEYalxvLQZQM4LzPe22WJiAtaCnT9PtYJXdi3G+/NupA/fXcYRWUnmPLsF9zx95VsKSpr/cki0mEp0DupgADDdeeksuS+sdw/qR9fbDvEpMeX8fO311Jc1voXWYtIx6ORiwBwsPwETy3ewj++2EloUAA/vKg3d4zuRUSIVjqIdCSaoYvLth84yh8+2MiC9ftIjA7lJxefxXez0wjsCAdORUQzdHFdr26RzL3lHN6YMYrULuE8+NZaLn1iKUs2FuGtf/xFxDUKdGlSdkZX3rzrfP5609mcqK7ltpdWcvPzX7KusIlvYBGRDkGBLs0yxnDZkCQW/fgi/vvKgWzYc4Qrn17OT+blUVhyvPUdiEi70gxdXFZ6vIpnPtnKC59tB+AHF/Ti7nG9iQnTCZVE2osOiopbFZYc508LN/F2XiFx4cHMntCXm0amExKkX/hEPE0HRcWtUuLC+fMNWbw780IGJsfw63c3cMlfPmX+2r06cCriRQp0OW2DU2L5x+0jefG2cwkNCuTuV1Zx3TOfk7uziW+ZERGPU6DLGTHGMK5fIvPvGc0frhtCweHjXPfMCmb8Xy7bDxz1dnkinYpm6OJWxyqreW7ZduZ+upXK6lpuGtmT2RP6Eh+lLzwQcQcdFJV2V1x2gsc/2sxrK3cTERzIjLG9uf3CXoQFt+M52EX8kA6KSrtLiA7lkWuGsPDe0YzMjOexhZsY97+f8EZuATW1OnAq4gkKdPGoPonRPHdrNq9NP4/E6FDu+9dqrnhqOcu+KfZ2aSJ+R4Eu7eK8zHjevvsCnpiSRVlFFbc8/xXTXviK/L1HvF2aiN9QoEu7CQgwXJ2Vwsc/vYhfXD6A1btLuOzJZdz/r9XsK63wdnkiPk8HRcVrSo5V8vTiLby8YicBAXDHhZnMGNubqFCdg12kOVrlIh3a7kPH+OPCTby7eg/xkSHcO7EvU0b0JDhQv0CKNKZVLtKhpXWN4Kmpw/nPjy6gd2IUv/zPeiY9vpT/5BVy9ES1t8sT8Rnq0KVDsdbyUX4Rv/8gn63FRwkJCuCC3vFcPLAHEwckkhgT5u0SRbxKIxfxOdU1tazccZhFG/azKH8fuw85zr+elRbHxQO7c/HA7vRNjMIYfTWedC4KdPFp1lo27S/jow37WbRhP6sLHN+alB4fwcUDHOF+TnoXgjRzl05AgS5+ZV9pBR/lO8J9xdaDVNbU0iUimHH9E7lkYHdG900gUitlxE8p0MVvlZ+oZunmYhZt2M/ijUWUHq8iJCiAC/t04+KB3ZnQX3N38S8tBbraGPFpUaFBXDYkicuGJFFVU8vKHYf4aEMRi/L3sXhjEfDt3P2Sgd3po7m7+DGXOnRjzGTgCSAQeM5a+/tGj18N/BaoBaqBe621y1vapzp08aS6ufui9ftZlL+fNc65e0Z8BBM1dxcfdkYjF2NMILAZuBgoAFYCU621G+ptEwUctdZaY8xQ4HVrbf+W9qtAl/bU3Nx9fH9HuI85qxsRIfqFVTq+Mx25jAC2WGu3OXf2GnA1cDLQrbXl9baPBHR+VOlQesSGcfN56dx8XjrlJ6r5dFMxizbsY9GGfby5qqDh3H1AIonRmruL73El0FOA3fVuFwAjG29kjLkGeBRIBC53S3UiHhAVGsTlQ5O4fOi3c/dFziWRizcWYYxj7j5xgObu4ltcGbl8F5hkrb3DefsWYIS1dlYz248BfmWtndjEY9OB6QA9e/Y8Z+fOnWdYvoj7WGvZuM+53r3R3N3xYaYenJPehcAAhbt4z5nO0EcBD1trJzlvPwRgrX20hedsB8611h5obhvN0KWj21t6nI/yi5xz9wNU1VjN3cXrzjTQg3AcFJ0AFOI4KHqjtXZ9vW36AFudB0XPBt4FUm0LO1egiy8pq6hi6eYDLNrgWA55pKKaUOfcfaLm7tKOzuigqLW22hgzE1iIY9niC0/OKVsAAAjDSURBVNba9caYGc7H5wLXAdOMMVXAceCGlsJcxNdEhwU3nLtvP8SHzrn7x/Xm7nXr3XsnaO4u7U+fFBU5A3Vz97qDqmsLHXP3Xt0imTggUXN3cTt99F+knTQ1d+8aGcL4/olcPLA7o/tq7i5nRoEu4gVlFVV86jzPzJJGc3fHevfuJESHertM8TEKdBEvazx3Lyw5jjEwPC2OiZq7Sxso0EU6EGst+XvLTp6KoP7c/aKzEshKiyMrLY70+AgFvJxCgS7Sge0tPc5HG/bz4Yb95Ow4zPGqGgC6RAQzzBnudZe4iBAvVyvepkAX8RHVNbVs3l9O3u4S8nYfJm93Cd8UlVP3Y9qrW2SDgB+QFENIkM4Y2Zko0EV8WFlFFWsLSvl6d4kz6EsoLjsBQEhQAIOSY04G/PC0LqR1Ddeoxo8p0EX8iLWWPaUV5O36totfW1hKRVUtAPGRIQ1GNcPS4ogND/Zy1eIu+sYiET9ijCElLpyUuHAuH5oEOFbRbNpXRt7uElY7u/glm4pOjmoyEyKdHXwcWWld6J8UTbC+3MPvqEMX8VNHnKOavN0lfL3LEfIHyh2jmtCgAAanxDaYx6d20ajGF2jkIiJYayksOe6YwzsDfm1hKSeqHaOablEh9QK+C0PTYokJ06imo9HIRUQwxpDaJYLULhFcMTQZ+HZU8/XJkD/MR/lFzu2hd0LUyTn88LQ4+vXQqKYjU4cuIg2UHq9iTcG3XXze7hIOHq0EICw4gMHJzlFNT0c3nxKnUU170shFRE6btZaCw8cbdPHr9hyh8uSoJtRxwNUZ8ENTY4nWqMZjNHIRkdNmjCGtawRpXSO4aphjVFNZXcvGfUcazOM/yt/v3B76OEc1dV18v+7RBGlU43Hq0EXELUqPVZFXUNJgffzhY1UAhAcHMiQl9mTAZ6XFkRQbplHNaVCHLiIeFxsRzEVnJXDRWQmAY1Sz69CxBssmX/psB5U1jlFNYrRjVDMoOZY+iVH0SYwio1sEoUGB3nwbPk2BLiIeYYwhPT6S9PhIrs5KARyjmvy9R04ebM3bXcKi/P0nPwAVYCA9PpLeCY6A750QeTLsNZdvnQJdRNpNSFAAw5zLIG913ne8soZtB8rZUlTO1qJythQ7rn+6uYiqmm9Hwt1jQh3hXhf2zqBPiArV6MZJgS4iXhUeEsig5FgGJcc2uL+6ppZdh46xpV7Iby0q543cAo5W1pzcLiYs6GQXf/KSEE1Kl/BO912uCnQR6ZCCAgPITIgiMyGKS+rdb61l35EKR9DXuyzeWMTrOQUntwsNcjy/flfv73N6BbqI+BRjDEmx4STFhjO6b0KDx0qOVTYM+uJyvt51mPfW7OkUc3oFuoj4jbiIELIzupKd0bXB/ccra9haXM7W4oZdvb/N6RXoIuL3wkMCGZwSy+CUU+f0O+vm9PUOyvrqnF4fLBIRaaS5Of3W4nIOlFee3M4bc3p9sEhEpA1amtMfPlrZcHTjnNO/u3rPyW28NadXoIuItEGXyBCyI89sTn/n6EzuGJ3p9toU6CIibtDcnL6q/np655w+ITrUIzUo0EVEPCg4MIDeCVH0Tohi0iDPvpbOZyki4icU6CIifkKBLiLiJ1wKdGPMZGPMJmPMFmPMg008fpMxZo3z8rkxZpj7SxURkZa0GujGmEBgDnApMBCYaowZ2Giz7cBF1tqhwG+BZ91dqIiItMyVDn0EsMVau81aWwm8BlxdfwNr7efW2sPOm18Aqe4tU0REWuNKoKcAu+vdLnDe15zbgQ+aesAYM90Yk2OMySkuLna9ShERaZUrgd7UmWeaPAGMMWYcjkB/oKnHrbXPWmuzrbXZCQkJTW0iIiKnyZUPFhUAafVupwJ7Gm9kjBkKPAdcaq092NpOc3NzDxhjdrpaaCPdgAOn+VxfpffcOeg9dw5n8p7Tm3ug1bMtGmOCgM3ABKAQWAncaK1dX2+bnsBiYJq19vPTLNJlxpic5s425q/0njsHvefOwVPvudUO3VpbbYyZCSwEAoEXrLXrjTEznI/PBX4FxAN/dZ4Evrqz/Q8SEfE2l87lYq2dD8xvdN/cetfvAO5wb2kiItIWvvpJ0c64zl3vuXPQe+4cPPKevfaNRSIi4l6+2qGLiEgjCnQRET/hc4He2onC/I0x5gVjTJExZp23a2kvxpg0Y8wSY0y+MWa9MeYeb9fkacaYMGPMV8aY1c73/Gtv19QejDGBxpivjTHvebuW9mCM2WGMWWuMyTPG5Lh9/740Q3eeKGwzcDGODzytBKZaazd4tTAPMsaMAcqBl621g71dT3swxiQBSdbaVcaYaCAX+I6f/382QKS1ttwYEwwsB+6x1n7h5dI8yhjzEyAbiLHWXuHtejzNGLMDyLbWeuSDVL7Wobd6ojB/Y61dChzydh3tyVq711q7ynm9DMin5fMH+TzrUO68Gey8+E63dRqMManA5Tg+YS5u4GuB3tYThYmPM8ZkAMOBL71biec5xw95QBGwyFrr7+/5ceBnQK23C2lHFvjQGJNrjJnu7p37WqC7fKIw8X3GmCjgTeBea+0Rb9fjadbaGmttFo7zJY0wxvjtiM0YcwVQZK3N9XYt7ewCa+3ZOL5f4kfOkarb+Fqgu3SiMPF9zjnym8Ar1tq3vF1Pe7LWlgCfAJO9XIonXQBc5ZwpvwaMN8b8w7sleZ61do/zzyLgbRxjZLfxtUBfCfQ1xvQyxoQAU4B3vFyTuJnzAOHzQL619s/erqc9GGMSjDFxzuvhwERgo3er8hxr7UPW2lRrbQaOn+PF1tqbvVyWRxljIp0H+THGRAKXAG5dveZTgW6trQbqThSWD7xe/6yP/sgY8yqwAuhnjCkwxtzu7ZrawQXALTi6tjzn5TJvF+VhScASY8waHI3LImttp1jK14l0B5YbY1YDXwHvW2sXuPMFfGrZooiINM+nOnQREWmeAl1ExE8o0EVE/IQCXUTETyjQRUT8hAJdRMRPKNBFRPzE/wfnGyMwI4ajtwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxV1bnw8d9DRkhCQkISyABhCEOYNSLOCEKpBREVxXqppSrSgnXoe6vX3ttyX733+lrHioKoiFpb61W5pV6rBBBRigpIlJmEBEgYkhBISIDMz/vH2cRDOJADJDlJzvP9fM7nnL33Wns/m2E966y9z16iqhhjjPE/HXwdgDHGGN+wBGCMMX7KEoAxxvgpSwDGGOOnLAEYY4yfCvR1AOeia9eumpKS4uswjDGmTdmwYcMhVY1tuL5NJYCUlBTWr1/v6zCMMaZNEZE9ntbbEJAxxvgprxKAiEwQkR0iki0ij3jY3kVElojIdyLytYgMdtYni8inIrJNRLaIyP1udeaKyD4RyXRe1zfdaRljjGlMo0NAIhIAvAiMA/KBdSKyVFW3uhV7FMhU1SkiMsApPxaoAX6lqt+ISASwQUQy3Oo+q6pPNeUJGWOM8Y431wBGAtmqmgMgIu8AkwH3BJAG/BeAqm4XkRQRiVfVA8ABZ32ZiGwDEhvUvSDV1dXk5+dTUVHRVLs0FyA0NJSkpCSCgoJ8HYoxphHeJIBEIM9tOR+4tEGZb4GbgC9EZCTQE0gCCk4WEJEUYATwlVu9OSLyE2A9rm8KRxoeXERmAjMBevTocVpw+fn5REREkJKSgoh4cTqmuagqxcXF5Ofn06tXL1+HY4xphDfXADy1qg2fIPcE0EVEMoH7gI24hn9cOxAJB94HHlDVo87q+UAfYDiubwlPezq4qi5U1XRVTY+NPe0uJioqKoiJibHGvxUQEWJiYuzbmDFthDffAPKBZLflJGC/ewGnUZ8BIK6WONd5ISJBuBr/t1X1A7c67t8OXgE+PL9TwBr/VsT+LoxpO7xJAOuAVBHpBewDpgE/di8gIlHAcVWtAu4GVqvqUScZvAZsU9VnGtTp7lwjAJgCbL6wUzHGmPajorqWXUXlZBe6XremJ5Mc3alJj9FoAlDVGhGZA3wCBACLVHWLiMxyti8ABgJvikgtrgu8dznVrwCmA5uc4SGAR1X1I+BJERmOazhpN3Bv052WMca0Dccqa9hVVE5WQTlZheVkF5aRVVjO3sPHOTldS0AHYUSPqJZPAABOg/1Rg3UL3D6vBVI91PsCz9cQUNXp5xSpoaamhsDANvXjbWOMo/REtdObL3Nr7MvZV3KivkxQgNC7aziDEyOZMiKRvnHhpMZFkNK1EyGBAU0ek7UmTeTGG28kLy+PiooK7r//fmbOnMnHH3/Mo48+Sm1tLV27dmXFihWUl5dz3333sX79ekSE3/3ud9x8882Eh4dTXl4OwHvvvceHH37I4sWL+elPf0p0dDQbN27koosu4rbbbuOBBx7gxIkTdOzYkddff53+/ftTW1vLww8/zCeffIKIcM8995CWlsa8efNYsmQJABkZGcyfP58PPvjgbKdijLkAh49VkVVQVt/AZxeWk1VYRsHRyvoyIYEd6BsXTnpKF26PS6ZvXASp8eH0jO5EYEDLPaChXSWAf//bFrbuP9p4wXOQltCZ300a1Gi5RYsWER0dzYkTJ7jkkkuYPHky99xzD6tXr6ZXr14cPnwYgMcee4zIyEg2bdoEwJEjp935epqdO3eyfPlyAgICOHr0KKtXryYwMJDly5fz6KOP8v7777Nw4UJyc3PZuHEjgYGBHD58mC5dujB79myKioqIjY3l9ddfZ8aMGRf2B2KMQVUpKqt0GndXA59V4Grsi49V1ZfrFBxAalw4V/aNJTU+nFSnR5/YpSMBHXx/w0S7SgC+9Ic//KG+p52Xl8fChQu5+uqr6++Hj46OBmD58uW888479fW6dOnS6L6nTp1KQIDr619paSl33nknWVlZiAjV1dX1+501a1b9ENHJ402fPp0//vGPzJgxg7Vr1/Lmm2820Rkb0/6pKgdKK1yNfEHZ9w1+QRlHK+rvdKdzaCCp8RGMS4t3DdvER5AaF073yNBWfWdcu0oA3vTUm8OqVatYvnw5a9eupVOnTowePZphw4axY8eO08qqqsd/EO7rGt5HHxYWVv/53/7t37j22mtZsmQJu3fvZvTo0Wfd74wZM5g0aRKhoaFMnTrVriEY40FdnZJ/5ISrJ19Y7vTmXQ3+sara+nIxYcH0jQvnhuEJpMa5Gvm+ceHERoS06ob+TKw1aAKlpaV06dKFTp06sX37dr788ksqKyv57LPPyM3NrR8Cio6OZvz48cybN4/nnnsOcA0BdenShfj4eLZt20b//v1ZsmQJERERZzxWYmIiAIsXL65fP378eBYsWMDo0aPrh4Cio6NJSEggISGBxx9/nIyMjGb/szCmNauprWPP4eOnNPBZheXsKiqnorquvlx85xBS4yKYmp7sXIh1NfQx4SE+jL7pWQJoAhMmTGDBggUMHTqU/v37M2rUKGJjY1m4cCE33XQTdXV1xMXFkZGRwb/+678ye/ZsBg8eTEBAAL/73e+46aabeOKJJ5g4cSLJyckMHjy4/oJwQ7/+9a+58847eeaZZxgzZkz9+rvvvpudO3cydOhQgoKCuOeee5gzZw4Ad9xxB0VFRaSlpbXIn4cxvlZZU8vuQ8frL8BmFZaTXVBO7qFjVNV+39AnRnWkb1w4l/WOITU+nL5xEfSNCyeyo388y0pUGz7VofVKT0/XhhPCbNu2jYEDB/ooorZhzpw5jBgxgrvuuqvxwk3A/k5MS3H/sZTr1kpXY7+n+Di1da62TQR6RHdyevGuYZvU+HD6xIYTFuIffWAR2aCq6Q3X+8fZ+7GLL76YsLAwnn7a46OWjGkzDpZWkJl3hG/zS9l50NXQ5x059cdSKTGd6BcXwY+GdK+/h753bBihQU1/D317YAmgnduwYYOvQzDmnB2rrGHTvlIy80rI3FtCZl4JB4+6bo4IChD6xIYzNCmSmy9KcoZuwkmJCSM40CY5PBeWAIwxPlVbp2QVlvFtnquh37i3hJ0FZTgjOPSM6cSlvaMZnhzF8OQo0hI6N8uvYv2RJQBjTIsqOFrBRqdXn5l3hE35pfW3WkZ2DGJYchTjB3VjRHIUw5KjiA4L9nHE7ZclAGNMszleVcOmfGcox3kdKP1+KGdg987cfHFSfe++V9ewNnk/fVtlCcAY0yTq6pTsonIy95aw0WnsdxaU1d+NkxzdkfSU74dyBiV0touzPmYJwBhzXgrLKuov0GbmlfBdfinlla7HI3QODWRYchTjBvZhmNPgt7cfUbUHlgBamPtTP41pK05U1bJ5f+kpDf7JxxgHdnAN5UwZkejq3feIoldMGB1awcPOzNlZAvBTNreAOZO6OmVXUXn9ME7m3hJ2uA3lJHXpyIgeUcy4IoURPaIYlBBpQzltlFctgIhMAJ7HNSPYq6r6RIPtXYBFuCZ5rwB+pqqbz1ZXRKKBvwApuGYEu1VVG3828tn8/RE4uOmCdnGabkPgh0+ccfPDDz9Mz549+cUvfgHA3LlzERFWr17NkSNHqK6u5vHHH2fy5MmNHqq8vJzJkyd7rPfmm2/y1FNPISIMHTqUt956i4KCAmbNmkVOTg4A8+fPJyEhgYkTJ7J5s2uGzaeeeory8nLmzp3L6NGjufzyy1mzZg033HAD/fr14/HHH6eqqoqYmBjefvtt4uPjPc5ZUFJSwubNm3n22WcBeOWVV9i2bRvPPPOM55MxbUZRWWX9HTmZeSV8l1dKmTOUExHiGsr5+TV9GO7clRMbYUM57UWjCUBEAoAXgXG4JohfJyJLVXWrW7FHgUxVnSIiA5zyYxup+wiwQlWfEJFHnOWHm/LkWsK0adN44IEH6hPAu+++y8cff8yDDz5I586dOXToEKNGjeKGG25o9O6G0NBQlixZclq9rVu38h//8R+sWbOGrl271s8t8Mtf/pJrrrmGJUuWUFtbS3l5eaPzC5SUlPDZZ58BrgfRffnll4gIr776Kk8++SRPP/20xzkLgoODGTp0KE8++SRBQUG8/vrrvPzyyxf6x2daWEV1LZudH1htdHr3J4dyAjoIA7pFcMPwBIYnRzGiRxS9u4bbUE475s03gJFAtqrmAIjIO8BkXHP/npQG/BeAqm4XkRQRiQd6n6XuZGC0U/8NYBUXmgDO0lNvLiNGjKCwsJD9+/dTVFREly5d6N69Ow8++CCrV6+mQ4cO7Nu3j4KCArp163bWfakqjz766Gn1Vq5cyS233ELXrl2B75/1v3Llyvrn+wcEBBAZGdloArjtttvqP+fn53Pbbbdx4MABqqqq6ucuONOcBWPGjOHDDz9k4MCBVFdXM2TIkHP80zItqa5OyTl07JTe/fYDZdQ4QzmJUR0ZnhzFTy9PYXiPKAYnRNIx2IZy/Ik3CSARyHNbzgcubVDmW+Am4AsRGQn0BJIaqRuvqgcAVPWAiMR5OriIzARmAvTo0cOLcFveLbfcwnvvvcfBgweZNm0ab7/9NkVFRWzYsIGgoCBSUlJOe8a/J2eqd6Zn/XsSGBhIXd33Tzs829wC9913Hw899BA33HADq1atYu7cucCZ5xa4++67+c///E8GDBhgM4u1QofKK8ncW8K3+d9fqC1zJi0JDwlkWHIk917Tm2FJrgu1cRGhPo7Y+Jo3CcBTy9PwEaJPAM+LSCawCdgI1HhZ96xUdSGwEFxPAz2Xui1l2rRp3HPPPRw6dIjPPvuMd999l7i4OIKCgvj000/Zs2ePV/spLS31WG/s2LFMmTKFBx98kJiYmPpn/Y8dO5b58+fzwAMPUFtby7Fjx4iPj6ewsJDi4mLCw8P58MMPmTBhwhmPd3JugTfeeKN+/ZnmLLj00kvJy8vjm2++4bvvvruQPzLTBPKPHGfFtkLW7T5MZl4J+Ue+H8rpHx/BpGHOUE5yFH1ibSjHnM6bBJAPJLstJwH73Quo6lFgBoC4uo65zqvTWeoWiEh3p/ffHSg8rzNoBQYNGkRZWRmJiYl0796dO+64g0mTJpGens7w4cMZMGCAV/s5U71Bgwbxm9/8hmuuuYaAgABGjBjB4sWLef7555k5cyavvfYaAQEBzJ8/n8suu4zf/va3XHrppfTq1eusx547dy5Tp04lMTGRUaNGkZubC3DGOQsAbr31VjIzM72aytI0LVVly/6jLNtaQMbWArYdcM1/nRAZyvAeUfzksp4MT+7C4MTOdAq2O7xM4xqdD0BEAoGdwFhgH7AO+LGqbnErEwUcV9UqEbkHuEpVf3K2uiLye6DY7SJwtKr++myx2HwAvjdx4kQefPBBxo4de8Yy9nfSdKpq6vgqt5iMrQUs31rA/tIKOghc3LML49LiGZfWjV5dwxrfkfFr5z0fgKrWiMgc4BNct3IuchrwWc72BcBA4E0RqcV1gfeus9V1dv0E8K6I3AXsBaZe6Ema5lNSUsLIkSMZNmzYWRt/c+HKKqpZtaOIjK0FfLqjkLKKGkKDOnBVaiwPjOvH2AFx9qta0yS8+p6oqh8BHzVYt8Dt81og1du6zvpiXN8M/M6mTZuYPn36KetCQkL46quvfBRR46Kioti5c6evw2i3DpSeYPnWApZtLeDLnGKqa5WYsGB+OLgb49K6cWXfrnaHjmly7WKg8FzukmkNhgwZQmZmpq/DaBZtaYpRX1JVth8sI8MZz9+0rxSAXl3D+NkVvRiXFs+IHl0IsAu3phm1+QQQGhpKcXExMTExbSoJtEeqSnFxMaGhdnuhJzW1dXy9+7BrPH9bAXmHTyACw5Oj+PWE/oxPi6dPbLj9OzYtps0ngKSkJPLz8ykqKvJ1KAZXQk5KSvJ1GK3GscoaVu90jeev2F5I6YlqggM7cGXfrvxidF/GDoyz+/GNz7T5BBAUFFT/C1ZjWoPCoxUs31ZIxtaDrNlVTFVNHVGdghg7MI7xafFclRpLWEib/69n2gH7V2jMBVJVsgvL6+/Pz8wrAVwToPzTpT0ZlxbPJSldCAywCctN62IJwJjzUFunbNhzhIytB8nYWsDu4uMADE2K5Ffj+jFuUDz94yNsPN+0apYAjPHSiapaPs9yjeev3F5I8bEqggKEy/p05a6renPdwDi6R3b0dZjGeM0SgDFncai8kpXbClm2tYAvsouoqK4jIjSQa/vHMX5QPNf0iyUiNMjXYRpzXiwBGNNATlF5/f35G/YeQdX1vJ3b0pMZl9aNkb2iCQ608XzT9lkCMH6vrk7JzC8hY2sBy7YcZFfRMQDSunfml2NSGZcWz6CEzjaeb9odSwDGL1VU1/KPXYecH2UVUlRWSUAH4dJe0Uwf1ZPr0uJJ6tLJ12Ea06wsARi/ceRYFSu3F5KxtYDVWUUcr6olLDiA0f3jGJcWz7X944jsZOP5xn9YAjDtWt7h4yxzhnbW7zlCbZ0S3zmEKSMSGZcWz2V9YggJtIesGf9kCcC0K6rKpn2l9Rdxtx8sA6BffDg/v6YP49LiGZIYabNjGYMlANNOFJVV8srnOSzN3M/Bo65JU9JTovnXHw1kXFo8PWNs0hRjGrIEYNq0kuNVvLw6h8VrdlNZU8t1A+P5P4P6M2ZAHNFhwb4Oz5hWzasEICITgOdxzer1qqo+0WB7JPBHoIezz6dU9XUR6Q/8xa1ob+C3qvqciMwF7gFOPsbzUWfyGGMadbSimtc+z2XRF7mUV9Vww7AE7h+bSu/YcF+HZkyb0WgCEJEA4EVgHK4J4teJyFJV3epWbDawVVUniUgssENE3lbVHcBwt/3sA5a41XtWVZ9qonMxfuB4VQ2L/7Gblz/LofRENRMGdePBcf3o3y3C16EZ0+Z48w1gJJCtqjkAIvIOMBnX3L8nKRAhrl/KhAOHgZoG+xkL7FLVPRcctfE7FdW1vP3VXuavyuZQeRVjBsTx0Lh+DE6M9HVoxrRZ3iSARCDPbTkfuLRBmXnAUmA/EAHcpqp1DcpMA/7cYN0cEfkJsB74laoeaXhwEZkJzATo0aOHF+Ga9qSqpo6/rM9j3sosCo5WckXfGF4e15+Le3bxdWjGtHneJABP98s1nPj1B0AmMAboA2SIyOeqehRARIKBG4B/caszH3jM2ddjwNPAz047kOpCYCFAenq6TTjrJ2pq6/hg4z6eX57FvpITpPfswnO3jeCyPjG+Ds2YdsObBJAPJLstJ+Hq6bubATyhrhnBs0UkFxgAfO1s/yHwjaoWnKzg/llEXgE+PPfwTXtTW6d8+N1+nlueRe6hYwxNiuQ/bxrC1ald7Vk8xjQxbxLAOiBVRHrhuog7DfhxgzJ7cY3xfy4i8UB/IMdt++00GP4Rke6qesBZnAJsPvfwTXuhqnyy5SDPZOxkZ0E5A7pFsHD6xYxLi7eG35hm0mgCUNUaEZkDfILrNtBFqrpFRGY52xfgGsJZLCKbcA0ZPayqhwBEpBOuO4jubbDrJ0VkOK4hoN0eths/oKp8uqOQp5ftZMv+o/SODWPej0dw/eDu9mtdY5qZuEZt2ob09HRdv369r8MwTUBV+ceuYp5atoONe0voEd2J+8emMnl4gs2da0wTE5ENqprecL39Eti0uHW7D/P0sh18mXOY7pGh/NdNQ7jl4iSCrOE3pkVZAjAt5tu8Ep7O2MnqnUV0DQ9h7qQ0po3sQWiQPY3TGF+wBGCa3bYDR3kmYycZWwvo0imIR68fwPRRKXQMtobfGF+yBGCaTXZhGc8uz+J/vztARGggvxrXjxlX9iI8xP7ZGdMa2P9E0+T2FB/j+RVZ/M/GfXQMCuC+MX25+8reNtuWMa2MJQDTZPaVnGDeyiz+e30+AR2Eu6/qzb1X9yYmPMTXoRljPLAEYC5Y4dEKXlq1iz99tReAfxrVk1+M7kNc51AfR2aMORtLAOa8HT5WxYLPdvHm2t1U1yq3picxZ0wqiVEdfR2aMcYLlgDMOSs9Uc2rn+ew6ItcTlTXcuOIRO4fm2rTLhrTxlgCMF4rr6zh9S9yWfh5DmUVNfxoaHcevC6VvnE2GYsxbZElANOoE1W1vPXlbuav2sWR49VcNzCeh8b1Iy2hs69DM8ZcAEsA5owqa2r581d7eXHVLorKKrm6XywPjevH8OQoX4dmjGkClgDMaapr63hvQz4vrMhif2kFl/aK5qU7LuKSlGhfh2aMaUKWAEy92jrlr5n7eG55FnsPH2dEjyh+P3UYl/eJsWfyG9MOWQIw1NUpH20+wLMZO9lVdIxBCZ1Z9NN0ru0fZw2/Me2YJQA/pqos31bI08t2sP1gGf3iw1nwTxcxPq2bTcZijB+wBOCHVJXVWYd4ZtkOvs0vJSWmE89PG87EoQkEWMNvjN/wKgGIyATgeVxTQr6qqk802B4J/BHo4ezzKVV93dm2GygDaoGak7PSiEg08BcgBdeUkLeq6pELPiNzVl/mFPP0sh2s232ExKiOPHnLUG4akWizcBnjhxpNACISALyIa17ffGCdiCxV1a1uxWYDW1V1kojEAjtE5G1VrXK2X3tyjmA3jwArVPUJEXnEWX74Qk/IePbN3iM8vWwHa7KLie8cwmM3Dua29GSCA63hN8ZfefMNYCSQrao5ACLyDjAZcE8ACkSI64phOHAYqGlkv5OB0c7nN4BVWAJocpv3lfJMxk5Wbi8kJiyYf5uYxh2X2ixcxhjvEkAikOe2nA9c2qDMPGApsB+IAG5T1TpnmwLLRESBl1V1obM+XlUPAKjqARGJ83RwEZkJzATo0aOHF+EacD2T/4m/b+fvmw8S2TGIX0/oz52XpRBmk7EYYxzetAaergpqg+UfAJnAGKAPkCEin6vqUeAKVd3vNPAZIrJdVVd7G6CTMBYCpKenNzyu8aCuTrn7jfUcKK3ggetS+dmVvegcapOxGGNO5c0AcD6Q7LachKun724G8IG6ZAO5wAAAVd3vvBcCS3ANKQEUiEh3AOe98HxPwpzq4y0HySos579uGsID1/Wzxt8Y45E3CWAdkCoivUQkGJiGa7jH3V5gLICIxAP9gRwRCRORCGd9GDAe2OzUWQrc6Xy+E/jrhZyIcamrU/6wIovesWFcP6S7r8MxxrRijQ4BqWqNiMwBPsF1G+giVd0iIrOc7QuAx4DFIrIJ15DRw6p6SER6A0ucX5MGAn9S1Y+dXT8BvCsid+FKIFOb+Nz80vJtBWw/WMYztw6ze/pbq7o6qKtp5FULtdWnLtfVQF11g+UakAAIDIaAEAgMgYBgCAz1sM55t193G4eotp1h9fT0dF2/fr2vw2i1VJUb5q3haEU1Kx66pv3e268K1cehshwqy6Cq7PvPtZUeGs+TDaaHBrXWQ4NaV91guQZqPTTQZ9xfI/uqvz/CRwJOJgYnUbgnh8CQ77c1fA8MPcO2kNP3Ub8fD9sa1rWE1OxEZMPJ32C5s1tC2pFVO4vYtK+U/3fzkNbX+NfVQVW561XpNNhVZW6fy6HyqNtnZ1tVuVtDX/59vaZoRKUDdAh0XkHQIeD75YBAt22BzragU5cDQyAgyG37Gfbl7f4Cghov0yHw1GNKAGgt1FS5kl/9e4WHdZVQW3Xqe02l520VR0+v616mrvrC//xPCvCQbDwmGLdvNcGdIDjceYU5r3AICf/+s/v64HBXXXMKSwDthKrywoosEqM6MmVEUtPstLbm1N61p4b5tEa8rEHZ8u8bfm90CHT+I0e4XsHhENoZIhMhOML5D35ye7izzm19UMcGDbKHBjogyNVwdmhlSbItqatzJQOPSaLSi+RT4TkhNaxbU+HaVlkGxw99v676BFQdg+pj3sfcIchDonBPFh6SR0jE2cu18aRiCaCd+MeuYr7ZW8LjNw52/bq36hgczj1zw1151EPvukGZmhPeHTwgxK1h7uz63KkrdOnluZF2b9xDnOWTjXtgqA0JtAUdOkCHUAgK9W0cdbWu4cCqY87rZMfE+ey+/rTlY66yx/NO3XY+SeWUROH2raNh8gjxtL7BcgsmFUsA7cTzK7Lo1jmUqQNDYeXj8PVCqCg9c4WgTqf3pDsnujXSTmPuqZGub7wj2kUvyLRhHQK+71A0FU9J5WSyaKmkclqiCIOxcyHp4qY7TywBtAtf5RSTn7uTRalrCXnhDlfPfcBEGHILhEae3gMPDneNSRtjTtcsSaXOlQQuJKk0A2sF2rqinVS+9xtWh64gYJ/AkFvhygcgtr+vIzPGnNShQ9MnlSZgCaCt2vcNfPEMuu1DLtEgtiXfypBbfgNRyY3XNcYYLAG0LaqQ+xl8/ozrPTSSv0X+mOeOXsvfpk8Be9CbMeYcWIvRFtTVwY7/hS+ehX0bIDwexv1ftnS/mV8u/JZ//kF/e8qnMeacWavRmtVWw6b/hi+eg0M7XLdVTnwOht0OQaE89+Z6IjsG8ZPLevo6UmNMG2QJoDWqOg7fvAn/eAGO5kP8YLj5NUi7sf7unW0HjpKxtYAHrkslwp72aYw5D5YAWpMTR+DrV+Gr+XC8GHpcBhOfhdRxp/04at7KbMJDAplxeS8fBWuMaessAbQGZQdh7Yuw/nXXoxVSx8OVD0HPyzwWzy4s46PNB/jF6D5EdrLevzHm/FgC8KXDObDmD5D5J9fDtQZNgSsfhG5Dzlpt3spsOgYFcNeVvVsoUGNMe2QJwBcObnbd0bPlA9fDyYb/GC7/JcT0abRq7qFjLP12P3df1ZvoMHsEgzHm/FkCaEl71roa/qxPXI9juGwOXDYbIrp5vYsXP80mKKAD91xlvX9jzIXx6nm4IjJBRHaISLaIPOJhe6SI/E1EvhWRLSIyw1mfLCKfisg2Z/39bnXmisg+Ecl0Xtc33Wm1IqqwcxksmgCvT4B96+Haf4UHN8P4x86p8c87fJwlG/fx40t7EBsR0oxBG2P8QaPfAEQkAHgRGIdrgvh1IrJUVbe6FZsNbFXVSSISC+wQkbeBGuBXqvqNMzfwBhHJcKv7rKo+1aRn1FrU1sDW/3Hdw1+wCTonwYT/BxdNdz3Z7zy8tGoXASLce3XjQ0XGGNMYb4aARgLZqpoDICLvAJMB9wSgQIS4Jv8NBw4DNap6ADNvG9AAABO0SURBVDgAoKplIrINSGxQt32proBv/+S6uHskF7r2g8kvwZCpF/TY5P0lJ3hvQx63XZJMt0gfP4PdGNMueJMAEoE8t+V84NIGZeYBS4H9QARwm+qpc/aJSAowAvjKbfUcEfkJsB7XN4UjDQ8uIjOBmQA9evTwIlwfqSyD9Ytg7UtQfhASRsC4t1yPZW6Cmade/mwXqjDrGuv9G2Oahjctk6fpmRrOJP8DIBNIAIYD80Skc/0ORMKB94EHVPWos3o+0McpfwB42tPBVXWhqqaranpsbKwX4bawY4dcE7A8Owgyfut6DPP0/4F7PoW0G5qk8S88WsGf1+Vx80VJJHXp1ARBG2OMd98A8gH3Zwwn4erpu5sBPKGqCmSLSC4wAPhaRIJwNf5vq+oHJyuoasHJzyLyCvDh+Z2Cj5Tkwdp5sOGN7ydgufKhJp+xB2Dh6hxq65RfXGu9f2NM0/EmAawDUkWkF7APmAb8uEGZvcBY4HMRiQf6AznONYHXgG2q+ox7BRHp7lwjAJgCbD7/02hBRTthzXPw3V9cy808Acuh8kr++NUeJg9PoGfM+V08NsYYTxpNAKpaIyJzgE+AAGCRqm4RkVnO9gXAY8BiEdmEa8joYVU9JCJXAtOBTSKS6ezyUVX9CHhSRIbjGk7aDdzbxOfWtJwJWNj2oWvi8vS74PI5ENW81yVe/TyXypo6Zl/bt1mPY4zxP179EMxpsD9qsG6B2+f9wHgP9b7A8zUEVHX6OUXqCw0nYAmJhKt+BaN+DmFdm/3wR45V8dba3UwcmkCf2PBmP54xxr/YL4E98TQBy3X/Duk/g9DOjddvIq+vyeVYVS1zrPdvjGkGlgDcnTYBS4rrcczDfgxBLXvv/dGKal7/x24mDOpG/26tayJpY0z7YAkAXBOwbHzLNQFLaZ7HCVha2htrdlNWUcN9Y633b4xpHv6dABpOwJI8Cn70tOt5/OLx0kWLKK+s4bU1uVw3MI5BCZE+i8MY0775ZwJoOAFL33Fw1UPQ83JfRwbAW2v3UHK8mvvGpPo6FGNMO+ZfCaDhBCxpN7omYOk+1NeR1TteVcOrn+dwdb9YhiVH+TocY0w75h8J4AImYGlpf/pqL8XHqvjlGBv7N8Y0L/9IAOtehZ0fuyZfGTUbOnf3dUQeVVTXsnB1Dpf1jiE9JdrX4Rhj2jn/SADX/gbG/hY6te5G9d31eRSWVfL8tBG+DsUY4wf8IwGEt8KniDZQWVPL/FW7uCSlC6N6t+5EZYxpHy78WcWmSby/YR8HSiu4b0wq4sNbUI0x/sMSQCtQXVvHS6uyGZYcxVWpzf+MIWOMAUsArcL/bNxH/pET/HJMX+v9G2NajCUAH6utU15atYtBCZ0ZMyDO1+EYY/yIJQAf+/C7/eQeOmZj/8aYFmcJwIfq6pQXVmbTPz6C8Wnxvg7HGONnvEoAIjJBRHaISLaIPOJhe6SI/E1EvhWRLSIyo7G6IhItIhkikuW8d2maU2o7/r75INmF5cwZ05cOHaz3b4xpWY0mABEJAF4EfgikAbeLSFqDYrOBrao6DBgNPC0iwY3UfQRYoaqpwApn2W+4ev9Z9I4N4/ohrfOXycaY9s2bbwAjgWxVzVHVKuAdYHKDMgpEOJPAhwOHgZpG6k4G3nA+vwHceEFn0sYs31bA9oNlzLm2LwHW+zfG+IA3CSARyHNbznfWuZsHDAT2A5uA+1W1rpG68ap6AMB595tbYFRdY/89Yzpxw7AEX4djjPFT3iQAT91TbbD8AyATSACGA/NEpLOXdc9+cJGZIrJeRNYXFRWdS9VWa9XOIjbtK2X26L4EBth1eGOMb3jT+uQDyW7LSbh6+u5mAB+oSzaQCwxopG6BiHQHcN4LPR1cVReqarqqpsfGtv5n+jRGVfnDiiwSozoy5aKGX6SMMableJMA1gGpItJLRIKBacDSBmX2AmMBRCQe6A/kNFJ3KXCn8/lO4K8XciJtxZrsYjbuLeHno/sQZL1/Y4wPNfo0UFWtEZE5wCdAALBIVbeIyCxn+wLgMWCxiGzCNezzsKoeAvBU19n1E8C7InIXrgQytWlPrXX6w8osunUOZWp6kq9DMcb4Oa8eB62qHwEfNVi3wO3zfmC8t3Wd9cU43xr8xVc5xXyde5jfTUojJDDA1+EYY/ycjUG0oBdWZtM1PITbR/bwdSjGGGMJoKVs2HOEL7IPce/VvQkNst6/Mcb3LAG0kBdWZhEdFswdo6z3b4xpHSwBtIDv8ktYtaOIu67sRadg/5iF0xjT+lkCaAEvrMwmsmMQP7msp69DMcaYepYAmtm2A0fJ2FrAjCtSiAgN8nU4xhhTzxJAM5u3MpvwkEBmXN7L16EYY8wpLAE0o6yCMj7afIA7L+9JZCfr/RtjWhdLAM1o3qfZdAwK4K4re/s6FGOMOY0lgGaSe+gYf/t2P9NH9SQ6LNjX4RhjzGksATSTFz/NJjiwA3dfZb1/Y0zrZAmgGeQdPs6Sjfu4fWQPYiNCfB2OMcZ4ZAmgGby0ahcBItx7dR9fh2KMMWdkCaCJ7S85wXsb8rj1kiS6RYb6OhxjjDkjSwBNbMFnuwD4+ei+Po7EGGPOzhJAEyo8WsE76/K4+aIkEqM6+jocY4w5K0sATejl1TnU1im/sN6/MaYN8CoBiMgEEdkhItki8oiH7f8sIpnOa7OI1IpItIj0d1ufKSJHReQBp85cEdnntu36pj65lnSovJK3v9rD5OEJ9Ijp5OtwjDGmUY0+m1hEAoAXgXFAPrBORJaq6taTZVT198DvnfKTgAdV9TBwGBjutp99wBK33T+rqk810bn41Kuf51JZU8fsa633b4xpG7z5BjASyFbVHFWtAt4BJp+l/O3Anz2sHwvsUtU95x5m63bkWBVvrd3NxKEJ9IkN93U4xhjjFW8SQCKQ57ac76w7jYh0AiYA73vYPI3TE8McEflORBaJSJcz7HOmiKwXkfVFRUVehNvyFq3J5VhVLfeNsd6/Mabt8CYBiId1eoayk4A1zvDP9zsQCQZuAP7bbfV8oA+uIaIDwNOedqiqC1U1XVXTY2NjvQi3ZZWeqGbxmt38cHA3+sVH+DocY4zxmjcJIB9IdltOAvafoaynXj7AD4FvVLXg5ApVLVDVWlWtA17BNdTU5rzxj92UVdYwx3r/xpg2xpsEsA5IFZFeTk9+GrC0YSERiQSuAf7qYR+nXRcQke5ui1OAzd4G3VqUV9awaE0u1w2MY1BCpK/DMcaYc9LoXUCqWiMic4BPgABgkapuEZFZzvYFTtEpwDJVPeZe37kuMA64t8GunxSR4biGk3Z72N7qvbV2DyXHq7lvTKqvQzHGmHPWaAIAUNWPgI8arFvQYHkxsNhD3eNAjIf1088hzlbneFUNr36ew9X9YhmWHOXrcIwx5pzZL4HP05++2kvxsSruH2tj/8aYtskSwHmoqK7l5dU5XN4nhot7Rvs6HGOMOS+WAM7DX9blUVRWaWP/xpg2zRLAOaqsqWXBZ7u4JKULo3pb798Y03ZZAjhH72/Yx4HSCu4bk4qIp9/IGWNM22AJ4BxU19bx0qpshiVHcVVqV1+HY4wxF8QSwDlYsnEf+UdOcP/Yvtb7N8a0eZYAvFRTW8dLn2YzOLEz1/aP83U4xhhzwSwBeOnD7w6wu/g4c661sX9jTPtgCcALdXXKvE+z6R8fwfi0eF+HY4wxTcISgBf+vvkg2YXlzBnTlw4drPdvjGkfLAE0oq5OeWFlFr1jw7h+SPfGKxhjTBthCaARy7cVsP1gGfeN6UuA9f6NMe2IJYCzUFX+sDKLnjGdmDQ0wdfhGGNMk7IEcBardhSxed9RZo/uS2CA/VEZY9oXa9XO4GTvPzGqI1MuSvR1OMYY0+S8SgAiMkFEdohItog84mH7P4tIpvPaLCK1IhLtbNstIpucbevd6kSLSIaIZDnvXZrutC7cmuxiNu4t4eej+xBkvX9jTDvUaMsmIgHAi7gmdk8DbheRNPcyqvp7VR2uqsOBfwE+U9XDbkWudbanu617BFihqqnACme51fjDyiy6dQ5lanqSr0Mxxphm4U3XdiSQrao5qloFvANMPkv50yaAP4PJwBvO5zeAG72o0yK+zCnm69zDzLqmNyGBAb4OxxhjmoU3CSARyHNbznfWncaZAH4C8L7bagWWicgGEZnptj5eVQ8AOO8eH7AjIjNFZL2IrC8qKvIi3Av3wsosuoaHMG1kjxY5njHG+II3CcDTze96hrKTgDUNhn+uUNWLcA0hzRaRq88lQFVdqKrpqpoeGxt7LlXPy4Y9R1iTXcy9V/cmNMh6/8aY9subBJAPJLstJwH7z1B2Gg2Gf1R1v/NeCCzBNaQEUCAi3QGc90Lvw24+L6zMIjosmDtGWe/fGNO+eZMA1gGpItJLRIJxNfJLGxYSkUjgGuCvbuvCRCTi5GdgPLDZ2bwUuNP5fKd7PV/5Lr+EVTuKuOvKXnQKDvR1OMYY06wabeVUtUZE5gCfAAHAIlXdIiKznO0LnKJTgGWqesytejywxHl8ciDwJ1X92Nn2BPCuiNwF7AWmNsUJXYgXVmYT2TGIn1zW09ehGGNMs/Oqm6uqHwEfNVi3oMHyYmBxg3U5wLAz7LMYGOt9qM1r6/6jZGwt4MHr+hERGuTrcIwxptnZL5wc8z7NIiIkkJ9ekeLrUIwxpkVYAgCyCsr4++aD3Hl5CpEdrfdvjPEPlgCAeZ9m0zEogJ9d2cvXoRhjTIvx+wSQe+gYf/t2P9NH9SQ6LNjX4RhjTIvx+wTw4qfZBAd24O6revs6FGOMaVF+nQDyDh9nycZ93D6yB7ERIb4OxxhjWpRfJ4CXVmUT0EGYdU0fX4dijDEtzm8TwL6SE7y3IZ/b0pOJ7xzq63CMMabF+W0CePmzXQDMGm29f2OMf/LLBFB4tIJ31uVx80VJJEZ19HU4xhjjE36ZAF5enUNtnfKL0X19HYoxxviM3yWAQ+WVvP3VHiYPT6BHTCdfh2OMMT7jdwnglc9zqKqpY/a11vs3xvg3v0oAR45V8dbaPUwcmkCf2HBfh2OMMT7lVwlg0ZpcjlfVMmeM9f6NMcZvEkDpiWoWr9nNDwd3o198hK/DMcYYn/ObBPDGP3ZTVlljvX9jjHF4lQBEZIKI7BCRbBF5xMP2fxaRTOe1WURqRSRaRJJF5FMR2SYiW0Tkfrc6c0Vkn1u965vyxNyVV9bw2he5XDcwjkEJkc11GGOMaVManRJSRAKAF4FxQD6wTkSWqurWk2VU9ffA753yk4AHVfWwiIQAv1LVb5zJ4TeISIZb3WdV9akmPqfTvLl2N6UnqrlvTGpzH8oYY9oMb74BjASyVTVHVauAd4DJZyl/O/BnAFU9oKrfOJ/LgG1A4oWFfO5iw0O4NT2JYclRLX1oY4xptbxJAIlAnttyPmdoxEWkEzABeN/DthRgBPCV2+o5IvKdiCwSkS5n2OdMEVkvIuuLioq8CPd0U9OTefIWj3PTG2OM3/ImAYiHdXqGspOANap6+JQdiITjSgoPqOpRZ/V8oA8wHDgAPO1ph6q6UFXTVTU9NjbWi3CNMcZ4w5sEkA8kuy0nAfvPUHYazvDPSSIShKvxf1tVPzi5XlULVLVWVeuAV3ANNRljjGkh3iSAdUCqiPQSkWBcjfzShoVEJBK4Bvir2zoBXgO2qeozDcp3d1ucAmw+9/CNMcacr0bvAlLVGhGZA3wCBACLVHWLiMxyti9wik4BlqnqMbfqVwDTgU0ikumse1RVPwKeFJHhuIaTdgP3NsUJGWOM8Y6onmk4v/VJT0/X9evX+zoMY4xpU0Rkg6qmN1zvN78ENsYYcypLAMYY46csARhjjJ9qU9cARKQI2HOe1bsCh5ownLbAztk/2Dn7hws5556qetoPqdpUArgQIrLe00WQ9szO2T/YOfuH5jhnGwIyxhg/ZQnAGGP8lD8lgIW+DsAH7Jz9g52zf2jyc/abawDGGGNO5U/fAIwxxrixBGCMMX7KLxJAY3MatzfOBDuFIuIXT1g929zT7ZWIhIrI1yLyrXPO/+7rmFqKiASIyEYR+dDXsbQEEdktIpucudOb9GFo7f4agDOn8U7c5jQGbnef07i9EZGrgXLgTVUd7Ot4mpvzaPHu7nNPAze2879jAcJUtdyZc+ML4H5V/dLHoTU7EXkISAc6q+pEX8fT3ERkN5Cuqk3+wzd/+AZwrnMat3mquho43GjBdqK1zD3dktSl3FkMcl7tuzcHiEgS8CPgVV/H0h74QwLwek5j0/adYe7pdskZCskECoEMVW335ww8B/waqPN1IC1IgWUiskFEZjbljv0hAZzLnMamDTvD3NPtljOl6nBc07SOFJF2PdwnIhOBQlXd4OtYWtgVqnoR8ENgtjPE2yT8IQGcy5zGpo0609zT/kBVS4BVwAQfh9LcrgBucMbE3wHGiMgffRtS81PV/c57IbCEJpw/3R8SgFdzGpu262xzT7dXIhIrIlHO547AdcB230bVvFT1X1Q1SVVTcP0/Xqmq/+TjsJqViIQ5NzYgImHAeJpw/vR2nwBUtQY4OafxNuBdVd3i26ial4j8GVgL9BeRfBG5y9cxNbOTc0+PcW6VyxSR630dVDPrDnwqIt/h6uRkqKpf3BbpZ+KBL0TkW+Br4H9V9eOm2nm7vw3UGGOMZ+3+G4AxxhjPLAEYY4yfsgRgjDF+yhKAMcb4KUsAxhjjpywBGGOMn7IEYIwxfur/A2e5BQmjPztqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tf.random.set_seed(33)\n",
    "\n",
    "MODEL_DIR = os.path.join(LOGDIR, 'hub')\n",
    "shutil.rmtree(MODEL_DIR, ignore_errors=True)\n",
    "\n",
    "BATCH_SIZE = 300\n",
    "EPOCHS = 100\n",
    "EMBED_DIM = 10\n",
    "PATIENCE = 3\n",
    "\n",
    "hub_model = build_hub_model()\n",
    "\n",
    "hub_history = hub_model.fit(\n",
    "    X_train, Y_train,\n",
    "    epochs=EPOCHS,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    validation_data=(X_valid, Y_valid),\n",
    "    callbacks=[callbacks.EarlyStopping(patience=PATIENCE),\n",
    "               callbacks.TensorBoard(MODEL_DIR)],\n",
    ")\n",
    "\n",
    "pd.DataFrame(hub_history.history)[['loss', 'val_loss']].plot()\n",
    "pd.DataFrame(hub_history.history)[['accuracy', 'val_accuracy']].plot()\n",
    "\n",
    "hub_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright 2020 Google Inc. Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}